#include "arm_arch.h"

#if __ARM_MAX_ARCH__>=8
.arch	armv8.2-a+crypto
.text
.globl	unroll8_eor3_aes_gcm_enc_128_kernel
.type	unroll8_eor3_aes_gcm_enc_128_kernel,%function
.align	4
unroll8_eor3_aes_gcm_enc_128_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L128_enc_ret
	stp	d8, d9, [sp, #-80]!
	lsr	x9, x1, #3
	mov	x16, x4
	mov	x8, x5
	stp	d10, d11, [sp, #16]
	stp	d12, d13, [sp, #32]
	stp	d14, d15, [sp, #48]
	mov	x5, #0xc200000000000000
	stp	x5, xzr, [sp, #64]
	add	x10, sp, #64

	mov	x15, #0x100000000				//set up counter increment
	movi	v31.16b, #0x0
	mov	v31.d[1], x15
	mov	x5, x9
	ld1	{ v0.16b}, [x16]					//CTR block 0

	sub	x5, x5, #1	 	//byte_len - 1

	and	x5, x5, #0xffffffffffffff80		//number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	rev32	v30.16b, v0.16b				//set up reversed counter

	add	v30.4s, v30.4s, v31.4s		//CTR block 0

	rev32	v1.16b, v30.16b				//CTR block 1
	add	v30.4s, v30.4s, v31.4s		//CTR block 1

	rev32	v2.16b, v30.16b				//CTR block 2
	add	v30.4s, v30.4s, v31.4s		//CTR block 2

	rev32	v3.16b, v30.16b				//CTR block 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 3

	rev32	v4.16b, v30.16b				//CTR block 4
	add	v30.4s, v30.4s, v31.4s		//CTR block 4

	rev32	v5.16b, v30.16b				//CTR block 5
	add	v30.4s, v30.4s, v31.4s		//CTR block 5
	ldp	q26, q27, [x8, #0]				  	//load rk0, rk1

	rev32	v6.16b, v30.16b				//CTR block 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 6

	rev32	v7.16b, v30.16b				//CTR block 7
	add	v30.4s, v30.4s, v31.4s		//CTR block 7

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 0
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 0

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 0

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 1

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 1
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 1

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 1
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 1

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 2
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 1
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 2

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 2

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 2

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 3

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 3
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 3

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 3

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 3

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 4
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 3
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 4

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 4

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 4

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 5
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 5
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 5

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 5

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 6

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 6

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 6
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 7

	ld1	{ v19.16b}, [x3]
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 7

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 7

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 7
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 7

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	ldr	q27, [x8, #160]					//load rk10

	aese	v3.16b, v26.16b						//AES block 8k+11 - round 9
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
	aese	v2.16b, v26.16b						//AES block 8k+10 - round 9

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	aese	v6.16b, v26.16b						//AES block 8k+14 - round 9

	aese	v4.16b, v26.16b						//AES block 8k+12 - round 9
	add	x5, x5, x0
	aese	v0.16b, v26.16b						//AES block 8k+8 - round 9

	aese	v7.16b, v26.16b						//AES block 8k+15 - round 9
	aese	v5.16b, v26.16b						//AES block 8k+13 - round 9
	aese	v1.16b, v26.16b						//AES block 8k+9 - round 9

	add	x4, x0, x1, lsr #3		//end_input_ptr
	cmp	x0, x5				//check if we have <= 8 blocks
	b.ge	.L128_enc_tail						//handle tail

	ldp	q8, q9, [x0], #32			//AES block 0, 1 - load plaintext

	ldp	q10, q11, [x0], #32			//AES block 2, 3 - load plaintext

	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load plaintext

	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load plaintext
	cmp	x0, x5				//check if we have <= 8 blocks

.inst	0xce006d08	//eor3 v8.16b, v8.16b, v0.16b, v27.16b				//AES block 0 - result
	rev32	v0.16b, v30.16b				//CTR block 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8

.inst	0xce016d29	//eor3 v9.16b, v9.16b, v1.16b, v27.16b				//AES block 1 - result
	stp	q8, q9, [x2], #32			//AES block 0, 1 - store result

	rev32	v1.16b, v30.16b				//CTR block 9
.inst	0xce056dad	//eor3 v13.16b, v13.16b, v5.16b, v27.16b				//AES block 5 - result
	add	v30.4s, v30.4s, v31.4s		//CTR block 9

.inst	0xce026d4a	//eor3 v10.16b, v10.16b, v2.16b, v27.16b				//AES block 2 - result
.inst	0xce066dce	//eor3 v14.16b, v14.16b, v6.16b, v27.16b				//AES block 6 - result
.inst	0xce046d8c	//eor3 v12.16b, v12.16b, v4.16b, v27.16b				//AES block 4 - result

	rev32	v2.16b, v30.16b				//CTR block 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 10

.inst	0xce036d6b	//eor3 v11.16b, v11.16b, v3.16b, v27.16b				//AES block 3 - result
.inst	0xce076def	//eor3 v15.16b, v15.16b, v7.16b,v27.16b				//AES block 7 - result
	stp	q10, q11, [x2], #32			//AES block 2, 3 - store result

	rev32	v3.16b, v30.16b				//CTR block 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 11
	stp	q12, q13, [x2], #32			//AES block 4, 5 - store result

	stp	q14, q15, [x2], #32			//AES block 6, 7 - store result

	rev32	v4.16b, v30.16b				//CTR block 12
	add	v30.4s, v30.4s, v31.4s		//CTR block 12
	b.ge	.L128_enc_prepretail					//do prepretail

.L128_enc_main_loop:	//main	loop start
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	rev64	v9.16b, v9.16b						//GHASH block 8k+1
	rev64	v8.16b, v8.16b						//GHASH block 8k
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0

	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k
	rev64	v13.16b, v13.16b						//GHASH block 8k+5 (t0, t1, t2 and t3 free)
	rev64	v11.16b, v11.16b						//GHASH block 8k+3

	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev32	v7.16b, v30.16b				//CTR block 8k+15

	rev64	v15.16b, v15.16b						//GHASH block 8k+7 (t0, t1, t2 and t3 free)

	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low

	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high

	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h3l | h3h
	ext	v25.16b, v25.16b, v25.16b, #8
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1

.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b,v9.16b			//GHASH block 8k+2, 8k+3 - high
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1

	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid

	rev64	v14.16b, v14.16b						//GHASH block 8k+6 (t0, t1, and t2 free)
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low

	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	rev64	v12.16b, v12.16b						//GHASH block 8k+4 (t0, t1, and t2 free)

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h1l | h1h
	ext	v22.16b, v22.16b, v22.16b, #8
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4

	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5

	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5

	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5

.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	ldr	d16, [x10]			//MODULO - load modulo constant
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6

	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6

	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
	ldp	q8, q9, [x0], #32			//AES block 8k+8, 8k+9 - load plaintext

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	rev32	v20.16b, v30.16b					//CTR block 8k+16
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+16

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6

.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
	ldp	q28, q26, [x8, #128]				//load rk8, rk9
.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	ldp	q10, q11, [x0], #32			//AES block 8k+10, 8k+11 - load plaintext

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7

	pmull	v21.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7

	rev32	v22.16b, v30.16b					//CTR block 8k+17
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	ldp	q12, q13, [x0], #32			//AES block 8k+12, 8k+13 - load plaintext
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+17

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	ldr	q27, [x8, #160]					//load rk10

	ext	v29.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment
	rev32	v23.16b, v30.16b					//CTR block 8k+18
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+18
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8

	aese	v2.16b, v26.16b						//AES block 8k+10 - round 9
	aese	v4.16b, v26.16b						//AES block 8k+12 - round 9
	aese	v1.16b, v26.16b						//AES block 8k+9 - round 9

	ldp	q14, q15, [x0], #32			//AES block 8k+14, 8k+15 - load plaintext
	rev32	v25.16b, v30.16b					//CTR block 8k+19
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+19

	cmp	x0, x5				//.LOOP CONTROL
.inst	0xce046d8c	//eor3 v12.16b, v12.16b, v4.16b, v27.16b				//AES block 4 - result
	aese	v7.16b, v26.16b						//AES block 8k+15 - round 9

	aese	v6.16b, v26.16b						//AES block 8k+14 - round 9
	aese	v3.16b, v26.16b						//AES block 8k+11 - round 9

.inst	0xce026d4a	//eor3 v10.16b, v10.16b, v2.16b, v27.16b				//AES block 8k+10 - result

	mov	v2.16b, v23.16b					//CTR block 8k+18
	aese	v0.16b, v26.16b						//AES block 8k+8 - round 9

	rev32	v4.16b, v30.16b				//CTR block 8k+20
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+20

.inst	0xce076def	//eor3 v15.16b, v15.16b, v7.16b, v27.16b				//AES block 7 - result
	aese	v5.16b, v26.16b						//AES block 8k+13 - round 9
	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low

.inst	0xce016d29	//eor3 v9.16b, v9.16b, v1.16b, v27.16b				//AES block 8k+9 - result
.inst	0xce036d6b	//eor3 v11.16b, v11.16b, v3.16b, v27.16b				//AES block 8k+11 - result
	mov	v3.16b, v25.16b					//CTR block 8k+19

	ext	v21.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment
.inst	0xce056dad	//eor3 v13.16b, v13.16b, v5.16b, v27.16b				//AES block 5 - result
	mov	v1.16b, v22.16b					//CTR block 8k+17

.inst	0xce006d08	//eor3 v8.16b, v8.16b, v0.16b, v27.16b				//AES block 8k+8 - result
	mov	v0.16b, v20.16b					//CTR block 8k+16
	stp	q8, q9, [x2], #32			//AES block 8k+8, 8k+9 - store result

	stp	q10, q11, [x2], #32			//AES block 8k+10, 8k+11 - store result
.inst	0xce066dce	//eor3 v14.16b, v14.16b, v6.16b, v27.16b				//AES block 6 - result

	stp	q12, q13, [x2], #32			//AES block 8k+12, 8k+13 - store result
.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low

	stp	q14, q15, [x2], #32			//AES block 8k+14, 8k+15 - store result
	b.lt	.L128_enc_main_loop

.L128_enc_prepretail:	//PREPRETAIL
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0

	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	rev64	v8.16b, v8.16b						//GHASH block 8k
	rev64	v9.16b, v9.16b						//GHASH block 8k+1

	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h6k | h5k
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13
	rev64	v11.16b, v11.16b						//GHASH block 8k+3

	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1

	rev32	v6.16b, v30.16b				//CTR block 8k+14

	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	rev64	v13.16b, v13.16b						//GHASH block 8k+5 (t0, t1, t2 and t3 free)
	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid

	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid

	rev64	v12.16b, v12.16b						//GHASH block 8k+4 (t0, t1, and t2 free)
	rev64	v15.16b, v15.16b						//GHASH block 8k+7 (t0, t1, t2 and t3 free)

	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid

	rev32	v7.16b, v30.16b				//CTR block 8k+15

	rev64	v14.16b, v14.16b						//GHASH block 8k+6 (t0, t1, and t2 free)

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0

	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0

.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0

	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1

.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	ldp	q27, q28, [x8, #64]				//load rk4, rk5

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h1l | h1h
	ext	v22.16b, v22.16b, v22.16b, #8
	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3

	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3

	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high

	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high

.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4

	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	ldr	d16, [x10]			//MODULO - load modulo constant

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	pmull	v21.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	ext	v29.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	ext	v18.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
.inst	0xce114a73	//eor3 v19.16b, v19.16b, v17.16b, v18.16b		 	//MODULO - fold into low
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8

	ldr	q27, [x8, #160]					//load rk10
	aese	v6.16b, v26.16b						//AES block 8k+14 - round 9
	aese	v2.16b, v26.16b						//AES block 8k+10 - round 9

	aese	v0.16b, v26.16b						//AES block 8k+8 - round 9
	aese	v1.16b, v26.16b						//AES block 8k+9 - round 9

	aese	v3.16b, v26.16b						//AES block 8k+11 - round 9
	aese	v5.16b, v26.16b						//AES block 8k+13 - round 9

	aese	v4.16b, v26.16b						//AES block 8k+12 - round 9
	aese	v7.16b, v26.16b						//AES block 8k+15 - round 9
.L128_enc_tail:	//TAIL

	sub	x5, x4, x0 	//main_end_input_ptr is number of bytes left to process
	ldr	q8, [x0], #16				//AES block 8k+8 - load plaintext

	mov	v29.16b, v27.16b
	ldp	q20, q21, [x3, #128]			//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8

.inst	0xce007509	//eor3 v9.16b, v8.16b, v0.16b, v29.16b			//AES block 8k+8 - result
	ext	v16.16b, v19.16b, v19.16b, #8				//prepare final partial tag
	ldp	q22, q23, [x3, #160]			//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	ext	v23.16b, v23.16b, v23.16b, #8

	ldp	q24, q25, [x3, #192]			//load h8k | h7k
	ext	v25.16b, v25.16b, v25.16b, #8
	cmp	x5, #112
	b.gt	.L128_enc_blocks_more_than_7

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	movi	v17.8b, #0

	cmp	x5, #96
	sub	v30.4s, v30.4s, v31.4s
	mov	v5.16b, v4.16b

	mov	v4.16b, v3.16b
	mov	v3.16b, v2.16b
	mov	v2.16b, v1.16b

	movi	v19.8b, #0
	movi	v18.8b, #0
	b.gt	.L128_enc_blocks_more_than_6

	mov	v7.16b, v6.16b
	cmp	x5, #80

	sub	v30.4s, v30.4s, v31.4s
	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	mov	v4.16b, v3.16b
	mov	v3.16b, v1.16b
	b.gt	.L128_enc_blocks_more_than_5

	cmp	x5, #64
	sub	v30.4s, v30.4s, v31.4s

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b

	mov	v5.16b, v4.16b
	mov	v4.16b, v1.16b
	b.gt	.L128_enc_blocks_more_than_4

	mov	v7.16b, v6.16b
	sub	v30.4s, v30.4s, v31.4s
	mov	v6.16b, v5.16b

	mov	v5.16b, v1.16b
	cmp	x5, #48
	b.gt	.L128_enc_blocks_more_than_3

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b
	mov	v6.16b, v1.16b

	cmp	x5, #32
	ldr	q24, [x3, #96]					//load h4k | h3k
	b.gt	.L128_enc_blocks_more_than_2

	cmp	x5, #16

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v1.16b
	b.gt	.L128_enc_blocks_more_than_1

	ldr	q21, [x3, #48]					//load h2k | h1k
	sub	v30.4s, v30.4s, v31.4s
	b	.L128_enc_blocks_less_than_1
.L128_enc_blocks_more_than_7:	//blocks	left >  7
	st1	{ v9.16b}, [x2], #16				//AES final-7 block  - store result

	rev64	v8.16b, v9.16b						//GHASH final-7 block
	ldr	q9, [x0], #16				//AES final-6 block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-7 block - mid

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH final-7 block - high

	ins	v18.d[0], v24.d[1]					//GHASH final-7 block - mid

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-7 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in

.inst	0xce017529	//eor3 v9.16b, v9.16b, v1.16b, v29.16b			//AES final-6 block - result

	pmull	v18.1q, v27.1d, v18.1d				//GHASH final-7 block - mid
	pmull	v19.1q, v8.1d, v25.1d				//GHASH final-7 block - low
.L128_enc_blocks_more_than_6:	//blocks	left >  6

	st1	{ v9.16b}, [x2], #16				//AES final-6 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-6 block
	ldr	q9, [x0], #16				//AES final-5 block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-6 block - mid

.inst	0xce027529	//eor3 v9.16b, v9.16b, v2.16b, v29.16b			//AES final-5 block - result
	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-6 block - low

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-6 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-6 block - mid
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-6 block - high

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-6 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-6 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-6 block - high
.L128_enc_blocks_more_than_5:	//blocks	left >  5

	st1	{ v9.16b}, [x2], #16				//AES final-5 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-5 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-5 block - mid
	ldr	q9, [x0], #16				//AES final-4 block - load plaintext
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-5 block - high

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-5 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-5 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-5 block - mid

.inst	0xce037529	//eor3 v9.16b, v9.16b, v3.16b, v29.16b			//AES final-4 block - result
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-5 block - low
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-5 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-5 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-5 block - mid
.L128_enc_blocks_more_than_4:	//blocks	left >  4

	st1	{ v9.16b}, [x2], #16			  	//AES final-4 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-4 block

	ldr	q9, [x0], #16				//AES final-3 block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-4 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final-4 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-4 block - mid

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final-4 block - low

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-4 block - high
	pmull	v27.1q, v27.1d, v21.1d				//GHASH final-4 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-4 block - low

.inst	0xce047529	//eor3 v9.16b, v9.16b, v4.16b, v29.16b			//AES final-3 block - result
	eor	v18.16b, v18.16b, v27.16b				//GHASH final-4 block - mid
.L128_enc_blocks_more_than_3:	//blocks	left >  3

	st1	{ v9.16b}, [x2], #16			  	//AES final-3 block - store result

	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8

	rev64	v8.16b, v9.16b						//GHASH final-3 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	movi	v16.8b, #0						//supress further partial tag feed in

	ins	v27.d[0], v8.d[1]					//GHASH final-3 block - mid
	ldr	q24, [x3, #96]				//load h4k | h3k
	pmull	v26.1q, v8.1d, v25.1d				//GHASH final-3 block - low

	ldr	q9, [x0], #16				//AES final-2 block - load plaintext

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-3 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-3 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-3 block - low

.inst	0xce057529	//eor3 v9.16b, v9.16b, v5.16b, v29.16b			//AES final-2 block - result

	pmull2	v27.1q, v27.2d, v24.2d				//GHASH final-3 block - mid
	pmull2	v28.1q, v8.2d, v25.2d				//GHASH final-3 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-3 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-3 block - high
.L128_enc_blocks_more_than_2:	//blocks	left >  2

	st1	{ v9.16b}, [x2], #16			  	//AES final-2 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-2 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ldr	q9, [x0], #16				//AES final-1 block - load plaintext

	ins	v27.d[0], v8.d[1]					//GHASH final-2 block - mid
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-2 block - mid
.inst	0xce067529	//eor3 v9.16b, v9.16b, v6.16b, v29.16b			//AES final-1 block - result

	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-2 block - high

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-2 block - low
	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-2 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-2 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-2 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-2 block - low
.L128_enc_blocks_more_than_1:	//blocks	left >  1

	st1	{ v9.16b}, [x2], #16			  	//AES final-1 block - store result

	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	rev64	v8.16b, v9.16b						//GHASH final-1 block
	ldr	q9, [x0], #16				//AES final block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	movi	v16.8b, #0						//supress further partial tag feed in
	ins	v27.d[0], v8.d[1]					//GHASH final-1 block - mid
.inst	0xce077529	//eor3 v9.16b, v9.16b, v7.16b, v29.16b			//AES final block - result

	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-1 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-1 block - mid

	ldr	q21, [x3, #48]				//load h2k | h1k

	ins	v27.d[1], v27.d[0]					//GHASH final-1 block - mid

	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-1 block - low
	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-1 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-1 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-1 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-1 block - low
.L128_enc_blocks_less_than_1:	//blocks	left <= 1

	rev32	v30.16b, v30.16b
	str	q30, [x16]					//store the updated counter
	and	x1, x1, #127			 	//bit_length %= 128

	sub	x1, x1, #128			 	//bit_length -= 128

	neg	x1, x1				//bit_length = 128 - #bits in input (in range [1,128])

	mvn	x6, xzr						//temp0_x = 0xffffffffffffffff
	ld1	{ v26.16b}, [x2]					//load existing bytes where the possibly partial last block is to be stored
	and	x1, x1, #127			 	//bit_length %= 128

	lsr	x6, x6, x1				//temp0_x is mask for top 64b of last block
	mvn	x7, xzr						//temp1_x = 0xffffffffffffffff
	cmp	x1, #64

	csel	x13, x7, x6, lt
	csel	x14, x6, xzr, lt

	mov	v0.d[1], x14
	mov	v0.d[0], x13					//ctr0b is mask for last block

	and	v9.16b, v9.16b, v0.16b					//possibly partial last block has zeroes in highest bits

	rev64	v8.16b, v9.16b						//GHASH final block

	bif	v9.16b, v26.16b, v0.16b					//insert existing bytes in top end of result before storing
	st1	{ v9.16b}, [x2]				//store all 16B

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v16.d[0], v8.d[1]					//GHASH final block - mid

	eor	v16.8b, v16.8b, v8.8b				//GHASH final block - mid
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8

	pmull	v16.1q, v16.1d, v21.1d				//GHASH final block - mid

	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final block - high
	eor	v18.16b, v18.16b, v16.16b				//GHASH final block - mid
	ldr	d16, [x10]			//MODULO - load modulo constant

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final block - low

	eor	v17.16b, v17.16b, v28.16b					//GHASH final block - high

	eor	v19.16b, v19.16b, v26.16b					//GHASH final block - low

	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment
	pmull	v29.1q, v17.1d, v16.1d		  	//MODULO - top 64b align with mid

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		  	//MODULO - karatsuba tidy up

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b		 	//MODULO - fold into mid

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ext	v21.16b, v18.16b, v18.16b, #8			  	//MODULO - other mid alignment

.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		  	//MODULO - fold into low
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	st1	{ v19.16b }, [x3]
	mov	x0, x9

	ldp	d10, d11, [sp, #16]
	ldp	d12, d13, [sp, #32]
	ldp	d14, d15, [sp, #48]
	ldp	d8, d9, [sp], #80
	ret

.L128_enc_ret:
	mov	w0, #0x0
	ret
.size	unroll8_eor3_aes_gcm_enc_128_kernel,.-unroll8_eor3_aes_gcm_enc_128_kernel
.globl	unroll8_eor3_aes_gcm_dec_128_kernel
.type	unroll8_eor3_aes_gcm_dec_128_kernel,%function
.align	4
unroll8_eor3_aes_gcm_dec_128_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L128_dec_ret
	stp	d8, d9, [sp, #-80]!
	lsr	x9, x1, #3
	mov	x16, x4
	mov	x8, x5
	stp	d10, d11, [sp, #16]
	stp	d12, d13, [sp, #32]
	stp	d14, d15, [sp, #48]
	mov	x5, #0xc200000000000000
	stp	x5, xzr, [sp, #64]
	add	x10, sp, #64

	mov	x5, x9
	ld1	{ v0.16b}, [x16]					//CTR block 0

	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	sub	x5, x5, #1		//byte_len - 1

	mov	x15, #0x100000000				//set up counter increment
	movi	v31.16b, #0x0
	mov	v31.d[1], x15
	ld1	{ v19.16b}, [x3]
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b

	rev32	v30.16b, v0.16b				//set up reversed counter

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 0

	add	v30.4s, v30.4s, v31.4s		//CTR block 0

	rev32	v1.16b, v30.16b				//CTR block 1
	add	v30.4s, v30.4s, v31.4s		//CTR block 1

	and	x5, x5, #0xffffffffffffff80	//number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	rev32	v2.16b, v30.16b				//CTR block 2
	add	v30.4s, v30.4s, v31.4s		//CTR block 2
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 0

	rev32	v3.16b, v30.16b				//CTR block 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 3

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 1

	rev32	v4.16b, v30.16b				//CTR block 4
	add	v30.4s, v30.4s, v31.4s		//CTR block 4

	rev32	v5.16b, v30.16b				//CTR block 5
	add	v30.4s, v30.4s, v31.4s		//CTR block 5

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 0

	rev32	v6.16b, v30.16b				//CTR block 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 0

	rev32	v7.16b, v30.16b				//CTR block 7

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 0
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 1

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 0

	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 1

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 2
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 1

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 2
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 2

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 2
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 2

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 3
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 3

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 3
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 3

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 3
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 3

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 4
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 3

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 4

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 4

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 5
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 5

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 5
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 5

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 5

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 5

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 6
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 5

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 6
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 6

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 6

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 7
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 7

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 7
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 7
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 7
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 7

	add	x5, x5, x0
	add	v30.4s, v30.4s, v31.4s		//CTR block 7

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 8

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 8

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 8

	aese	v0.16b, v26.16b						//AES block 0 - round 9
	aese	v1.16b, v26.16b						//AES block 1 - round 9
	aese	v6.16b, v26.16b						//AES block 6 - round 9

	ldr	q27, [x8, #160]					//load rk10
	aese	v4.16b, v26.16b						//AES block 4 - round 9
	aese	v3.16b, v26.16b						//AES block 3 - round 9

	aese	v2.16b, v26.16b						//AES block 2 - round 9
	aese	v5.16b, v26.16b						//AES block 5 - round 9
	aese	v7.16b, v26.16b						//AES block 7 - round 9

	add	x4, x0, x1, lsr #3		//end_input_ptr
	cmp	x0, x5				//check if we have <= 8 blocks
	b.ge	.L128_dec_tail						//handle tail

	ldp	q8, q9, [x0], #32			//AES block 0, 1 - load ciphertext

.inst	0xce006d00	//eor3 v0.16b, v8.16b, v0.16b, v27.16b				//AES block 0 - result
.inst	0xce016d21	//eor3 v1.16b, v9.16b, v1.16b, v27.16b				//AES block 1 - result
	stp	q0, q1, [x2], #32			//AES block 0, 1 - store result

	rev32	v0.16b, v30.16b				//CTR block 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8
	ldp	q10, q11, [x0], #32			//AES block 2, 3 - load ciphertext

	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load ciphertext

	rev32	v1.16b, v30.16b				//CTR block 9
	add	v30.4s, v30.4s, v31.4s		//CTR block 9
	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load ciphertext

.inst	0xce036d63	//eor3 v3.16b, v11.16b, v3.16b, v27.16b				//AES block 3 - result
.inst	0xce026d42	//eor3 v2.16b, v10.16b, v2.16b, v27.16b				//AES block 2 - result
	stp	q2, q3, [x2], #32			//AES block 2, 3 - store result

	rev32	v2.16b, v30.16b				//CTR block 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 10

.inst	0xce066dc6	//eor3 v6.16b, v14.16b, v6.16b, v27.16b				//AES block 6 - result

	rev32	v3.16b, v30.16b				//CTR block 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 11

.inst	0xce046d84	//eor3 v4.16b, v12.16b, v4.16b, v27.16b				//AES block 4 - result
.inst	0xce056da5	//eor3 v5.16b, v13.16b, v5.16b, v27.16b				//AES block 5 - result
	stp	q4, q5, [x2], #32			//AES block 4, 5 - store result

.inst	0xce076de7	//eor3 v7.16b, v15.16b, v7.16b, v27.16b				//AES block 7 - result
	stp	q6, q7, [x2], #32			//AES block 6, 7 - store result
	rev32	v4.16b, v30.16b				//CTR block 12

	cmp	x0, x5				//check if we have <= 8 blocks
	add	v30.4s, v30.4s, v31.4s		//CTR block 12
	b.ge	.L128_dec_prepretail					//do prepretail

.L128_dec_main_loop:	//main	loop start
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8

	rev64	v9.16b, v9.16b						//GHASH block 8k+1
	rev64	v8.16b, v8.16b						//GHASH block 8k
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0

	rev64	v14.16b, v14.16b						//GHASH block 8k+6
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8

	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	rev64	v12.16b, v12.16b						//GHASH block 8k+4
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k

	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high
	rev64	v11.16b, v11.16b						//GHASH block 8k+3

	rev32	v7.16b, v30.16b				//CTR block 8k+15
	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	rev64	v13.16b, v13.16b						//GHASH block 8k+5

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8

	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	rev64	v15.16b, v15.16b						//GHASH block 8k+7
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid

	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3

	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high

	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4

	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5

	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b 			//GHASH block 8k+4, 8k+5 - mid
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	ldr	d16, [x10]			//MODULO - load modulo constant
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7

	rev32	v20.16b, v30.16b					//CTR block 8k+16
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+16

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	rev32	v22.16b, v30.16b					//CTR block 8k+17

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment
	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+17

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	ldp	q8, q9, [x0], #32			//AES block 8k+8, 8k+9 - load ciphertext

	ldp	q10, q11, [x0], #32			//AES block 8k+10, 8k+11 - load ciphertext
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	rev32	v23.16b, v30.16b					//CTR block 8k+18

	ldp	q12, q13, [x0], #32			//AES block 8k+12, 8k+13 - load ciphertext
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	ldp	q14, q15, [x0], #32			//AES block 8k+14, 8k+15 - load ciphertext
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+18

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8

	aese	v0.16b, v26.16b						//AES block 8k+8 - round 9
	aese	v1.16b, v26.16b						//AES block 8k+9 - round 9
	ldr	q27, [x8, #160]					//load rk10

	aese	v6.16b, v26.16b						//AES block 8k+14 - round 9
	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v2.16b, v26.16b						//AES block 8k+10 - round 9

	aese	v7.16b, v26.16b						//AES block 8k+15 - round 9
	aese	v4.16b, v26.16b						//AES block 8k+12 - round 9
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment

	rev32	v25.16b, v30.16b					//CTR block 8k+19
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+19

	aese	v3.16b, v26.16b						//AES block 8k+11 - round 9
	aese	v5.16b, v26.16b						//AES block 8k+13 - round 9
.inst	0xce016d21	//eor3 v1.16b, v9.16b, v1.16b, v27.16b				//AES block 8k+9 - result

.inst	0xce006d00	//eor3 v0.16b, v8.16b, v0.16b, v27.16b				//AES block 8k+8 - result
.inst	0xce076de7	//eor3 v7.16b, v15.16b, v7.16b, v27.16b				//AES block 8k+15 - result
.inst	0xce066dc6	//eor3 v6.16b, v14.16b, v6.16b, v27.16b				//AES block 8k+14 - result

.inst	0xce026d42	//eor3 v2.16b, v10.16b, v2.16b, v27.16b				//AES block 8k+10 - result
	stp	q0, q1, [x2], #32			//AES block 8k+8, 8k+9 - store result
	mov	v1.16b, v22.16b					//CTR block 8k+17

.inst	0xce046d84	//eor3 v4.16b, v12.16b, v4.16b, v27.16b				//AES block 8k+12 - result
.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	mov	v0.16b, v20.16b					//CTR block 8k+16

.inst	0xce036d63	//eor3 v3.16b, v11.16b, v3.16b, v27.16b				//AES block 8k+11 - result
	cmp	x0, x5				//.LOOP CONTROL
	stp	q2, q3, [x2], #32			//AES block 8k+10, 8k+11 - store result

.inst	0xce056da5	//eor3 v5.16b, v13.16b, v5.16b, v27.16b				//AES block 8k+13 - result
	mov	v2.16b, v23.16b					//CTR block 8k+18

	stp	q4, q5, [x2], #32			//AES block 8k+12, 8k+13 - store result
	rev32	v4.16b, v30.16b				//CTR block 8k+20
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+20

	stp	q6, q7, [x2], #32			//AES block 8k+14, 8k+15 - store result
	mov	v3.16b, v25.16b					//CTR block 8k+19
	b.lt	.L128_dec_main_loop

.L128_dec_prepretail:	//PREPRETAIL
	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0
	rev64	v8.16b, v8.16b						//GHASH block 8k

	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1

	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev64	v9.16b, v9.16b						//GHASH block 8k+1

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	rev64	v13.16b, v13.16b						//GHASH block 8k+5

	rev64	v12.16b, v12.16b						//GHASH block 8k+4

	rev64	v14.16b, v14.16b						//GHASH block 8k+6

	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k
	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0

	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	rev32	v7.16b, v30.16b				//CTR block 8k+15
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0

.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k - mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0

	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	rev64	v15.16b, v15.16b						//GHASH block 8k+7
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3

	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4

	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5

	ldr	d16, [x10]			//MODULO - load modulo constant
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7
	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	ldr	q27, [x8, #160]					//load rk10

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8

	aese	v6.16b, v26.16b						//AES block 8k+14 - round 9
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8

.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15
	aese	v2.16b, v26.16b						//AES block 8k+10 - round 9

	aese	v3.16b, v26.16b						//AES block 8k+11 - round 9
	aese	v5.16b, v26.16b						//AES block 8k+13 - round 9
	aese	v0.16b, v26.16b						//AES block 8k+8 - round 9

	aese	v4.16b, v26.16b						//AES block 8k+12 - round 9
	aese	v1.16b, v26.16b						//AES block 8k+9 - round 9
	aese	v7.16b, v26.16b						//AES block 8k+15 - round 9

.L128_dec_tail:	//TAIL

	mov	v29.16b, v27.16b
	sub	x5, x4, x0 	//main_end_input_ptr is number of bytes left to process

	cmp	x5, #112

	ldp	q24, q25, [x3, #192]			//load h8k | h7k
	ext	v25.16b, v25.16b, v25.16b, #8
	ldr	q9, [x0], #16				//AES block 8k+8 - load ciphertext

	ldp	q20, q21, [x3, #128]			//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ext	v16.16b, v19.16b, v19.16b, #8				//prepare final partial tag

	ldp	q22, q23, [x3, #160]			//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	ext	v23.16b, v23.16b, v23.16b, #8

.inst	0xce00752c	//eor3 v12.16b, v9.16b, v0.16b, v29.16b				//AES block 8k+8 - result
	b.gt	.L128_dec_blocks_more_than_7

	cmp	x5, #96
	mov	v7.16b, v6.16b
	movi	v19.8b, #0

	movi	v17.8b, #0
	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	mov	v4.16b, v3.16b
	mov	v3.16b, v2.16b
	mov	v2.16b, v1.16b

	movi	v18.8b, #0
	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L128_dec_blocks_more_than_6

	cmp	x5, #80
	sub	v30.4s, v30.4s, v31.4s

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	mov	v4.16b, v3.16b
	mov	v3.16b, v1.16b
	b.gt	.L128_dec_blocks_more_than_5

	cmp	x5, #64

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	mov	v4.16b, v1.16b
	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L128_dec_blocks_more_than_4

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b

	mov	v5.16b, v1.16b
	cmp	x5, #48
	b.gt	.L128_dec_blocks_more_than_3

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b
	cmp	x5, #32

	ldr	q24, [x3, #96]				//load h4k | h3k
	mov	v6.16b, v1.16b
	b.gt	.L128_dec_blocks_more_than_2

	cmp	x5, #16

	mov	v7.16b, v1.16b
	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L128_dec_blocks_more_than_1

	sub	v30.4s, v30.4s, v31.4s
	ldr	q21, [x3, #48]				//load h2k | h1k
	b	.L128_dec_blocks_less_than_1
.L128_dec_blocks_more_than_7:	//blocks	left >  7
	rev64	v8.16b, v9.16b						//GHASH final-7 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v18.d[0], v24.d[1]					//GHASH final-7 block - mid

	pmull	v19.1q, v8.1d, v25.1d				//GHASH final-7 block - low
	ins	v27.d[0], v8.d[1]					//GHASH final-7 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in
	ldr	q9, [x0], #16				//AES final-6 block - load ciphertext

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-7 block - mid

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH final-7 block - high
	st1	{ v12.16b}, [x2], #16			 	//AES final-7 block  - store result
.inst	0xce01752c	//eor3 v12.16b, v9.16b, v1.16b, v29.16b				//AES final-6 block - result

	pmull	v18.1q, v27.1d, v18.1d			 	//GHASH final-7 block - mid
.L128_dec_blocks_more_than_6:	//blocks	left >  6

	rev64	v8.16b, v9.16b						//GHASH final-6 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-6 block - mid

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-6 block - mid

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-6 block - low
	ldr	q9, [x0], #16				//AES final-5 block - load ciphertext
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-6 block - mid
	st1	{ v12.16b}, [x2], #16			 	//AES final-6 block - store result
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-6 block - high

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-6 block - low
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-6 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-6 block - mid
.inst	0xce02752c	//eor3 v12.16b, v9.16b, v2.16b, v29.16b				//AES final-5 block - result
.L128_dec_blocks_more_than_5:	//blocks	left >  5

	rev64	v8.16b, v9.16b						//GHASH final-5 block

	ldr	q9, [x0], #16				//AES final-4 block - load ciphertext
	st1	{ v12.16b}, [x2], #16			 	//AES final-5 block - store result

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-5 block - mid

.inst	0xce03752c	//eor3 v12.16b, v9.16b, v3.16b, v29.16b				//AES final-4 block - result

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-5 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-5 block - mid
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-5 block - low
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-5 block - mid
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-5 block - high
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-5 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-5 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-5 block - high
.L128_dec_blocks_more_than_4:	//blocks	left >  4

	rev64	v8.16b, v9.16b						//GHASH final-4 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	ldr	q9, [x0], #16				//AES final-3 block - load ciphertext

	ins	v27.d[0], v8.d[1]					//GHASH final-4 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final-4 block - high

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final-4 block - low

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-4 block - high

	st1	{ v12.16b}, [x2], #16			 	//AES final-4 block - store result
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-4 block - mid

.inst	0xce04752c	//eor3 v12.16b, v9.16b, v4.16b, v29.16b				//AES final-3 block - result
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-4 block - low

	pmull	v27.1q, v27.1d, v21.1d				//GHASH final-4 block - mid

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-4 block - mid
.L128_dec_blocks_more_than_3:	//blocks	left >  3

	st1	{ v12.16b}, [x2], #16			 	//AES final-3 block - store result
	rev64	v8.16b, v9.16b						//GHASH final-3 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-3 block - mid

	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	ldr	q24, [x3, #96]				//load h4k | h3k

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-3 block - mid

	ldr	q9, [x0], #16				//AES final-2 block - load ciphertext

	ins	v27.d[1], v27.d[0]					//GHASH final-3 block - mid
	pmull	v26.1q, v8.1d, v25.1d				//GHASH final-3 block - low
	pmull2	v28.1q, v8.2d, v25.2d				//GHASH final-3 block - high

	movi	v16.8b, #0						//supress further partial tag feed in
.inst	0xce05752c	//eor3 v12.16b, v9.16b, v5.16b, v29.16b				//AES final-2 block - result
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-3 block - low

	pmull2	v27.1q, v27.2d, v24.2d				//GHASH final-3 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-3 block - high
	eor	v18.16b, v18.16b, v27.16b				//GHASH final-3 block - mid
.L128_dec_blocks_more_than_2:	//blocks	left >  2

	rev64	v8.16b, v9.16b						//GHASH final-2 block

	st1	{ v12.16b}, [x2], #16			 	//AES final-2 block - store result

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	movi	v16.8b, #0						//supress further partial tag feed in

	ins	v27.d[0], v8.d[1]					//GHASH final-2 block - mid

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-2 block - mid

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-2 block - low

	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-2 block - high
	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-2 block - mid
	ldr	q9, [x0], #16				//AES final-1 block - load ciphertext

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-2 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-2 block - low

.inst	0xce06752c	//eor3 v12.16b, v9.16b, v6.16b, v29.16b				//AES final-1 block - result
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-2 block - high
.L128_dec_blocks_more_than_1:	//blocks	left >  1

	st1	{ v12.16b}, [x2], #16			 	//AES final-1 block - store result
	rev64	v8.16b, v9.16b						//GHASH final-1 block

	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	movi	v16.8b, #0						//supress further partial tag feed in

	ins	v27.d[0], v8.d[1]					//GHASH final-1 block - mid

	ldr	q9, [x0], #16				//AES final block - load ciphertext
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-1 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-1 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-1 block - high
	ldr	q21, [x3, #48]				//load h2k | h1k

	ins	v27.d[1], v27.d[0]					//GHASH final-1 block - mid
.inst	0xce07752c	//eor3 v12.16b, v9.16b, v7.16b, v29.16b				//AES final block - result

	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-1 block - low

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-1 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-1 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-1 block - mid
.L128_dec_blocks_less_than_1:	//blocks	left <= 1

	and	x1, x1, #127				//bit_length %= 128

	sub	x1, x1, #128				//bit_length -= 128

	neg	x1, x1				//bit_length = 128 - #bits in input (in range [1,128])

	mvn	x6, xzr						//temp0_x = 0xffffffffffffffff
	and	x1, x1, #127				//bit_length %= 128

	lsr	x6, x6, x1				//temp0_x is mask for top 64b of last block
	cmp	x1, #64
	mvn	x7, xzr						//temp1_x = 0xffffffffffffffff

	csel	x13, x7, x6, lt
	csel	x14, x6, xzr, lt

	mov	v0.d[1], x14
	mov	v0.d[0], x13					//ctr0b is mask for last block

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ld1	{ v26.16b}, [x2]					//load existing bytes where the possibly partial last block is to be stored

	and	v9.16b, v9.16b, v0.16b					//possibly partial last block has zeroes in highest bits

	rev64	v8.16b, v9.16b						//GHASH final block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final block - high
	ins	v16.d[0], v8.d[1]					//GHASH final block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final block - high
	eor	v16.8b, v16.8b, v8.8b				//GHASH final block - mid

	bif	v12.16b, v26.16b, v0.16b					//insert existing bytes in top end of result before storing

	pmull	v16.1q, v16.1d, v21.1d				//GHASH final block - mid
	st1	{ v12.16b}, [x2]				//store all 16B

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final block - low

	eor	v18.16b, v18.16b, v16.16b				//GHASH final block - mid
	ldr	d16, [x10]			//MODULO - load modulo constant

	eor	v19.16b, v19.16b, v26.16b					//GHASH final block - low

	eor	v14.16b, v17.16b, v19.16b				//MODULO - karatsuba tidy up

	pmull	v21.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	ext	v17.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment

	eor	v18.16b, v18.16b, v14.16b				//MODULO - karatsuba tidy up

.inst	0xce115652	//eor3 v18.16b, v18.16b, v17.16b, v21.16b			//MODULO - fold into mid

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ext	v18.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment

.inst	0xce124673	//eor3 v19.16b, v19.16b, v18.16b, v17.16b			//MODULO - fold into low
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	st1	{ v19.16b }, [x3]
	rev32	v30.16b, v30.16b

	str	q30, [x16]					//store the updated counter

	mov	x0, x9

	ldp	d10, d11, [sp, #16]
	ldp	d12, d13, [sp, #32]
	ldp	d14, d15, [sp, #48]
	ldp	d8, d9, [sp], #80
	ret
.L128_dec_ret:
	mov	w0, #0x0
	ret
.size	unroll8_eor3_aes_gcm_dec_128_kernel,.-unroll8_eor3_aes_gcm_dec_128_kernel
.globl	unroll8_eor3_aes_gcm_enc_192_kernel
.type	unroll8_eor3_aes_gcm_enc_192_kernel,%function
.align	4
unroll8_eor3_aes_gcm_enc_192_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L192_enc_ret
	stp	d8, d9, [sp, #-80]!
	lsr	x9, x1, #3
	mov	x16, x4
	mov	x8, x5
	stp	d10, d11, [sp, #16]
	stp	d12, d13, [sp, #32]
	stp	d14, d15, [sp, #48]
	mov	x5, #0xc200000000000000
	stp	x5, xzr, [sp, #64]
	add	x10, sp, #64

	mov	x5, x9
	ld1	{ v0.16b}, [x16]					//CTR block 0

	mov	x15, #0x100000000				//set up counter increment
	movi	v31.16b, #0x0
	mov	v31.d[1], x15

	rev32	v30.16b, v0.16b				//set up reversed counter

	add	v30.4s, v30.4s, v31.4s		//CTR block 0

	rev32	v1.16b, v30.16b				//CTR block 1
	add	v30.4s, v30.4s, v31.4s		//CTR block 1

	rev32	v2.16b, v30.16b				//CTR block 2
	add	v30.4s, v30.4s, v31.4s		//CTR block 2

	rev32	v3.16b, v30.16b				//CTR block 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 3

	rev32	v4.16b, v30.16b				//CTR block 4
	add	v30.4s, v30.4s, v31.4s		//CTR block 4
	sub	x5, x5, #1		//byte_len - 1

	and	x5, x5, #0xffffffffffffff80	//number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	rev32	v5.16b, v30.16b				//CTR block 5
	add	v30.4s, v30.4s, v31.4s		//CTR block 5
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1

	add	x5, x5, x0

	rev32	v6.16b, v30.16b				//CTR block 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 6

	rev32	v7.16b, v30.16b				//CTR block 7

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 0
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 0

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 0

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 1

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 1
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 1

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 2
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 1

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 1
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 2

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 2

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 2

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 3

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 3
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 3

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 3

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 3

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 4
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 3

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 4

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 4

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 5

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 5
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 5

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 5
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 5

	add	v30.4s, v30.4s, v31.4s		//CTR block 7

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 6
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 6

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 6
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 7

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 7

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 7

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 7
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 7

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 8

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 8

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 8

	add	x4, x0, x1, lsr #3		//end_input_ptr
	cmp	x0, x5				//check if we have <= 8 blocks
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 9

	ld1	{ v19.16b}, [x3]
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	ldp	q27, q28, [x8, #160]				//load rk10, rk11

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 9

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 9
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 9

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 9
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 9

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 14 - round 10
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 9
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 11 - round 10

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 9 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 13 - round 10
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 12 - round 10

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 10 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 15 - round 10

	aese	v6.16b, v28.16b						//AES block 14 - round 11
	aese	v3.16b, v28.16b						//AES block 11 - round 11

	aese	v4.16b, v28.16b						//AES block 12 - round 11
	aese	v7.16b, v28.16b						//AES block 15 - round 11
	ldr	q26, [x8, #192]					//load rk12

	aese	v1.16b, v28.16b						//AES block 9 - round 11
	aese	v5.16b, v28.16b						//AES block 13 - round 11

	aese	v2.16b, v28.16b						//AES block 10 - round 11
	aese	v0.16b, v28.16b						//AES block 8 - round 11
	b.ge	.L192_enc_tail						//handle tail

	ldp	q8, q9, [x0], #32			//AES block 0, 1 - load plaintext

	ldp	q10, q11, [x0], #32			//AES block 2, 3 - load plaintext

	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load plaintext

	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load plaintext

.inst	0xce006908	//eor3 v8.16b, v8.16b, v0.16b, v26.16b				//AES block 0 - result
	rev32	v0.16b, v30.16b				//CTR block 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8

.inst	0xce03696b	//eor3 v11.16b, v11.16b, v3.16b, v26.16b				//AES block 3 - result
.inst	0xce016929	//eor3 v9.16b, v9.16b, v1.16b, v26.16b				//AES block 1 - result

	rev32	v1.16b, v30.16b				//CTR block 9
	add	v30.4s, v30.4s, v31.4s		//CTR block 9
.inst	0xce04698c	//eor3 v12.16b, v12.16b, v4.16b, v26.16b				//AES block 4 - result

.inst	0xce0569ad	//eor3 v13.16b, v13.16b, v5.16b, v26.16b				//AES block 5 - result
.inst	0xce0769ef	//eor3 v15.16b, v15.16b, v7.16b, v26.16b				//AES block 7 - result
	stp	q8, q9, [x2], #32			//AES block 0, 1 - store result

.inst	0xce02694a	//eor3 v10.16b, v10.16b, v2.16b, v26.16b				//AES block 2 - result
	rev32	v2.16b, v30.16b				//CTR block 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 10

	stp	q10, q11, [x2], #32			//AES block 2, 3 - store result
	cmp	x0, x5				//check if we have <= 8 blocks

	rev32	v3.16b, v30.16b				//CTR block 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 11
.inst	0xce0669ce	//eor3 v14.16b, v14.16b, v6.16b, v26.16b				//AES block 6 - result

	stp	q12, q13, [x2], #32			//AES block 4, 5 - store result

	rev32	v4.16b, v30.16b				//CTR block 12
	stp	q14, q15, [x2], #32			//AES block 6, 7 - store result
	add	v30.4s, v30.4s, v31.4s		//CTR block 12

	b.ge	.L192_enc_prepretail					//do prepretail

.L192_enc_main_loop:	//main	loop start
	rev64	v12.16b, v12.16b						//GHASH block 8k+4 (t0, t1, and t2 free)
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	rev64	v10.16b, v10.16b						//GHASH block 8k+2

	rev32	v5.16b, v30.16b				//CTR block 8k+13
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8

	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0
	rev64	v8.16b, v8.16b						//GHASH block 8k
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8

	rev64	v9.16b, v9.16b						//GHASH block 8k+1
	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	rev64	v13.16b, v13.16b						//GHASH block 8k+5 (t0, t1, t2 and t3 free)

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0
	rev32	v7.16b, v30.16b				//CTR block 8k+15
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low

	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k

	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1

	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1

	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2

	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	ldp	q27, q28, [x8, #64]				//load rk4, rk5

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k - mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3

	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3

	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4

.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	rev64	v15.16b, v15.16b						//GHASH block 8k+7 (t0, t1, t2 and t3 free)

	rev64	v14.16b, v14.16b						//GHASH block 8k+6 (t0, t1, and t2 free)
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5

	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high

	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	ldp	q28, q26, [x8, #128]				//load rk8, rk9
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7

.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	ldr	d16, [x10]			//MODULO - load modulo constant
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7

	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	ldp	q27, q28, [x8, #160]				//load rk10, rk11

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
	rev32	v20.16b, v30.16b					//CTR block 8k+16
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+16

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9
	ldp	q8, q9, [x0], #32			//AES block 8k+8, 8k+9 - load plaintext

	pmull	v21.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	rev32	v22.16b, v30.16b					//CTR block 8k+17
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+17

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	ldr	q26, [x8, #192]					//load rk12
	ext	v29.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10
	ldp	q10, q11, [x0], #32			//AES block 8k+10, 8k+11 - load plaintext

	aese	v4.16b, v28.16b						//AES block 8k+12 - round 11
.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	ldp	q12, q13, [x0], #32			//AES block 8k+12, 8k+13 - load plaintext

	ldp	q14, q15, [x0], #32			//AES block 8k+14, 8k+15 - load plaintext
	aese	v2.16b, v28.16b						//AES block 8k+10 - round 11
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10

	rev32	v23.16b, v30.16b					//CTR block 8k+18
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10
	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10
	aese	v5.16b, v28.16b						//AES block 8k+13 - round 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+18

	aese	v7.16b, v28.16b						//AES block 8k+15 - round 11
	aese	v0.16b, v28.16b						//AES block 8k+8 - round 11
.inst	0xce04698c	//eor3 v12.16b, v12.16b, v4.16b, v26.16b				//AES block 4 - result

	aese	v6.16b, v28.16b						//AES block 8k+14 - round 11
	aese	v3.16b, v28.16b						//AES block 8k+11 - round 11
	aese	v1.16b, v28.16b						//AES block 8k+9 - round 11

	rev32	v25.16b, v30.16b					//CTR block 8k+19
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+19
.inst	0xce0769ef	//eor3 v15.16b, v15.16b, v7.16b, v26.16b				//AES block 7 - result

.inst	0xce02694a	//eor3 v10.16b, v10.16b, v2.16b, v26.16b				//AES block 8k+10 - result
.inst	0xce006908	//eor3 v8.16b, v8.16b, v0.16b, v26.16b				//AES block 8k+8 - result
	mov	v2.16b, v23.16b					//CTR block 8k+18

.inst	0xce016929	//eor3 v9.16b, v9.16b, v1.16b, v26.16b				//AES block 8k+9 - result
	mov	v1.16b, v22.16b					//CTR block 8k+17
	stp	q8, q9, [x2], #32			//AES block 8k+8, 8k+9 - store result
	ext	v21.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment

.inst	0xce0669ce	//eor3 v14.16b, v14.16b, v6.16b, v26.16b				//AES block 6 - result
	mov	v0.16b, v20.16b					//CTR block 8k+16
	rev32	v4.16b, v30.16b				//CTR block 8k+20

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+20
.inst	0xce0569ad	//eor3 v13.16b, v13.16b, v5.16b, v26.16b				//AES block 5 - result
.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low

.inst	0xce03696b	//eor3 v11.16b, v11.16b, v3.16b, v26.16b				//AES block 8k+11 - result
	mov	v3.16b, v25.16b					//CTR block 8k+19

	stp	q10, q11, [x2], #32			//AES block 8k+10, 8k+11 - store result

	stp	q12, q13, [x2], #32			//AES block 8k+12, 8k+13 - store result

	cmp	x0, x5				//.LOOP CONTROL
	stp	q14, q15, [x2], #32			//AES block 8k+14, 8k+15 - store result
	b.lt	.L192_enc_main_loop

.L192_enc_prepretail:	//PREPRETAIL
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v8.16b, v8.16b						//GHASH block 8k
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k

	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8

	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev32	v7.16b, v30.16b				//CTR block 8k+15
	rev64	v9.16b, v9.16b						//GHASH block 8k+1

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0

	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low

	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2
	rev64	v13.16b, v13.16b						//GHASH block 8k+5 (t0, t1, t2 and t3 free)
	rev64	v14.16b, v14.16b						//GHASH block 8k+6 (t0, t1, and t2 free)

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2

	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	ldp	q27, q28, [x8, #64]				//load rk4, rk5

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3

	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	rev64	v12.16b, v12.16b						//GHASH block 8k+4 (t0, t1, and t2 free)

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4

.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	rev64	v15.16b, v15.16b						//GHASH block 8k+7 (t0, t1, t2 and t3 free)
	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low

	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6
.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6
	ldr	d16, [x10]			//MODULO - load modulo constant
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6

	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	ext	v29.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	pmull	v21.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9
	ldp	q27, q28, [x8, #160]				//load rk10, rk11
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9

	ext	v21.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ldr	q26, [x8, #192]					//load rk12

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10

.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10

	aese	v1.16b, v28.16b						//AES block 8k+9 - round 11
	aese	v7.16b, v28.16b						//AES block 8k+15 - round 11

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	aese	v3.16b, v28.16b						//AES block 8k+11 - round 11

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15
	aese	v2.16b, v28.16b						//AES block 8k+10 - round 11
	aese	v0.16b, v28.16b						//AES block 8k+8 - round 11

	aese	v6.16b, v28.16b						//AES block 8k+14 - round 11
	aese	v4.16b, v28.16b						//AES block 8k+12 - round 11
	aese	v5.16b, v28.16b						//AES block 8k+13 - round 11

.L192_enc_tail:	//TAIL

	ldp	q20, q21, [x3, #128]			//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	sub	x5, x4, x0 	//main_end_input_ptr is number of bytes left to process

	ldr	q8, [x0], #16				//AES block 8k+8 - l3ad plaintext

	ldp	q24, q25, [x3, #192]			//load h8k | h7k
	ext	v25.16b, v25.16b, v25.16b, #8

	mov	v29.16b, v26.16b

	ldp	q22, q23, [x3, #160]			//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	ext	v23.16b, v23.16b, v23.16b, #8
	cmp	x5, #112

.inst	0xce007509	//eor3 v9.16b, v8.16b, v0.16b, v29.16b			//AES block 8k+8 - result
	ext	v16.16b, v19.16b, v19.16b, #8				//prepare final partial tag
	b.gt	.L192_enc_blocks_more_than_7

	cmp	x5, #96
	mov	v7.16b, v6.16b
	movi	v17.8b, #0

	mov	v6.16b, v5.16b
	movi	v19.8b, #0
	sub	v30.4s, v30.4s, v31.4s

	mov	v5.16b, v4.16b
	mov	v4.16b, v3.16b
	mov	v3.16b, v2.16b

	mov	v2.16b, v1.16b
	movi	v18.8b, #0
	b.gt	.L192_enc_blocks_more_than_6

	mov	v7.16b, v6.16b
	cmp	x5, #80

	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b
	mov	v4.16b, v3.16b

	mov	v3.16b, v1.16b
	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L192_enc_blocks_more_than_5

	cmp	x5, #64
	sub	v30.4s, v30.4s, v31.4s

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	mov	v4.16b, v1.16b
	b.gt	.L192_enc_blocks_more_than_4

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	mov	v5.16b, v1.16b

	sub	v30.4s, v30.4s, v31.4s
	cmp	x5, #48
	b.gt	.L192_enc_blocks_more_than_3

	mov	v7.16b, v6.16b
	mov	v6.16b, v1.16b
	sub	v30.4s, v30.4s, v31.4s

	ldr	q24, [x3, #96]				//load h4k | h3k
	cmp	x5, #32
	b.gt	.L192_enc_blocks_more_than_2

	sub	v30.4s, v30.4s, v31.4s

	cmp	x5, #16
	mov	v7.16b, v1.16b
	b.gt	.L192_enc_blocks_more_than_1

	sub	v30.4s, v30.4s, v31.4s
	ldr	q21, [x3, #48]				//load h2k | h1k
	b	.L192_enc_blocks_less_than_1
.L192_enc_blocks_more_than_7:	//blocks	left >  7
	st1	{ v9.16b}, [x2], #16			 	//AES final-7 block  - store result

	rev64	v8.16b, v9.16b						//GHASH final-7 block
	ins	v18.d[0], v24.d[1]					//GHASH final-7 block - mid

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-7 block - mid

	ldr	q9, [x0], #16				//AES final-6 block - load plaintext

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-7 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
	pmull	v19.1q, v8.1d, v25.1d				//GHASH final-7 block - low

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH final-7 block - high

	pmull	v18.1q, v27.1d, v18.1d			 	//GHASH final-7 block - mid
.inst	0xce017529	//eor3 v9.16b, v9.16b, v1.16b, v29.16b			//AES final-6 block - result
.L192_enc_blocks_more_than_6:	//blocks	left >  6

	st1	{ v9.16b}, [x2], #16			 	//AES final-6 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-6 block

	ldr	q9, [x0], #16				//AES final-5 block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-6 block - mid

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-6 block - low
.inst	0xce027529	//eor3 v9.16b, v9.16b, v2.16b, v29.16b			//AES final-5 block - result

	movi	v16.8b, #0						//supress further partial tag feed in
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-6 block - high
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-6 block - mid

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-6 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-6 block - high
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-6 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-6 block - mid
.L192_enc_blocks_more_than_5:	//blocks	left >  5

	st1	{ v9.16b}, [x2], #16			 	//AES final-5 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-5 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-5 block - mid

	ldr	q9, [x0], #16				//AES final-4 block - load plaintext
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-5 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-5 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-5 block - high

	ins	v27.d[1], v27.d[0]					//GHASH final-5 block - mid
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-5 block - low

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-5 block - low
	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-5 block - mid

.inst	0xce037529	//eor3 v9.16b, v9.16b, v3.16b, v29.16b			//AES final-4 block - result
	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-5 block - mid
.L192_enc_blocks_more_than_4:	//blocks	left >  4

	st1	{ v9.16b}, [x2], #16				//AES final-4 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-4 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ldr	q9, [x0], #16				//AES final-3 block - load plaintext
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final-4 block - high
	ins	v27.d[0], v8.d[1]					//GHASH final-4 block - mid

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final-4 block - low
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-4 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-4 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-4 block - low

	pmull	v27.1q, v27.1d, v21.1d				//GHASH final-4 block - mid

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-4 block - mid
.inst	0xce047529	//eor3 v9.16b, v9.16b, v4.16b, v29.16b			//AES final-3 block - result
.L192_enc_blocks_more_than_3:	//blocks	left >  3

	ldr	q24, [x3, #96]				//load h4k | h3k
	st1	{ v9.16b}, [x2], #16			 	//AES final-3 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-3 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	movi	v16.8b, #0						//supress further partial tag feed in

	ldr	q9, [x0], #16				//AES final-2 block - load plaintext
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8

	ins	v27.d[0], v8.d[1]					//GHASH final-3 block - mid

.inst	0xce057529	//eor3 v9.16b, v9.16b, v5.16b, v29.16b			//AES final-2 block - result
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-3 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-3 block - mid
	pmull	v26.1q, v8.1d, v25.1d				//GHASH final-3 block - low

	pmull2	v28.1q, v8.2d, v25.2d				//GHASH final-3 block - high
	pmull2	v27.1q, v27.2d, v24.2d				//GHASH final-3 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-3 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-3 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-3 block - high
.L192_enc_blocks_more_than_2:	//blocks	left >  2

	st1	{ v9.16b}, [x2], #16			 	//AES final-2 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-2 block
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ldr	q9, [x0], #16				//AES final-1 block - load plaintext
	ins	v27.d[0], v8.d[1]					//GHASH final-2 block - mid

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-2 block - mid

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-2 block - low
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-2 block - high
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-2 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-2 block - low
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-2 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-2 block - mid
.inst	0xce067529	//eor3 v9.16b, v9.16b, v6.16b, v29.16b			//AES final-1 block - result
.L192_enc_blocks_more_than_1:	//blocks	left >  1

	ldr	q22, [x3, #64]				//load h1l | h1h
	ext	v22.16b, v22.16b, v22.16b, #8
	st1	{ v9.16b}, [x2], #16			 	//AES final-1 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-1 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-1 block - mid
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-1 block - low

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-1 block - low
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-1 block - high
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-1 block - mid

	ldr	q9, [x0], #16				//AES final block - load plaintext
	ldr	q21, [x3, #48]				//load h2k | h1k

	ins	v27.d[1], v27.d[0]					//GHASH final-1 block - mid

.inst	0xce077529	//eor3 v9.16b, v9.16b, v7.16b, v29.16b			//AES final block - result
	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-1 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-1 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-1 block - high
.L192_enc_blocks_less_than_1:	//blocks	left <= 1

	mvn	x6, xzr						//temp0_x = 0xffffffffffffffff
	and	x1, x1, #127				//bit_length %= 128

	sub	x1, x1, #128				//bit_length -= 128

	neg	x1, x1				//bit_length = 128 - #bits in input (in range [1,128])

	and	x1, x1, #127				//bit_length %= 128

	lsr	x6, x6, x1				//temp0_x is mask for top 64b of last block
	cmp	x1, #64
	mvn	x7, xzr						//temp1_x = 0xffffffffffffffff

	csel	x13, x7, x6, lt
	csel	x14, x6, xzr, lt

	mov	v0.d[1], x14
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8

	ld1	{ v26.16b}, [x2]					//load existing bytes where the possibly partial last block is to be stored
	mov	v0.d[0], x13					//ctr0b is mask for last block

	and	v9.16b, v9.16b, v0.16b					//possibly partial last block has zeroes in highest bits

	rev64	v8.16b, v9.16b						//GHASH final block
	bif	v9.16b, v26.16b, v0.16b					//insert existing bytes in top end of result before storing

	st1	{ v9.16b}, [x2]				//store all 16B

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v16.d[0], v8.d[1]					//GHASH final block - mid
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final block - high

	eor	v17.16b, v17.16b, v28.16b					//GHASH final block - high
	pmull	v26.1q, v8.1d, v20.1d				//GHASH final block - low

	eor	v16.8b, v16.8b, v8.8b				//GHASH final block - mid

	pmull	v16.1q, v16.1d, v21.1d				//GHASH final block - mid

	eor	v18.16b, v18.16b, v16.16b				//GHASH final block - mid
	ldr	d16, [x10]			//MODULO - load modulo constant

	eor	v19.16b, v19.16b, v26.16b					//GHASH final block - low
	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment

	rev32	v30.16b, v30.16b

	str	q30, [x16]					//store the updated counter
.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up

	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment

.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	st1	{ v19.16b }, [x3]

	mov	x0, x9					//return sizes

	ldp	d10, d11, [sp, #16]
	ldp	d12, d13, [sp, #32]
	ldp	d14, d15, [sp, #48]
	ldp	d8, d9, [sp], #80
	ret

.L192_enc_ret:
	mov	w0, #0x0
	ret
.size	unroll8_eor3_aes_gcm_enc_192_kernel,.-unroll8_eor3_aes_gcm_enc_192_kernel
.globl	unroll8_eor3_aes_gcm_dec_192_kernel
.type	unroll8_eor3_aes_gcm_dec_192_kernel,%function
.align	4
unroll8_eor3_aes_gcm_dec_192_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L192_dec_ret
	stp	d8, d9, [sp, #-80]!
	lsr	x9, x1, #3
	mov	x16, x4
	mov	x8, x5
	stp	d10, d11, [sp, #16]
	stp	d12, d13, [sp, #32]
	stp	d14, d15, [sp, #48]
	mov	x5, #0xc200000000000000
	stp	x5, xzr, [sp, #64]
	add	x10, sp, #64

	mov	x5, x9
	ld1	{ v0.16b}, [x16]					//CTR block 0
	ld1	{ v19.16b}, [x3]

	mov	x15, #0x100000000			//set up counter increment
	movi	v31.16b, #0x0
	mov	v31.d[1], x15

	rev32	v30.16b, v0.16b				//set up reversed counter

	add	v30.4s, v30.4s, v31.4s		//CTR block 0

	rev32	v1.16b, v30.16b				//CTR block 1
	add	v30.4s, v30.4s, v31.4s		//CTR block 1

	rev32	v2.16b, v30.16b				//CTR block 2
	add	v30.4s, v30.4s, v31.4s		//CTR block 2

	rev32	v3.16b, v30.16b				//CTR block 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 3

	rev32	v4.16b, v30.16b				//CTR block 4
	add	v30.4s, v30.4s, v31.4s		//CTR block 4

	rev32	v5.16b, v30.16b				//CTR block 5
	add	v30.4s, v30.4s, v31.4s		//CTR block 5
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1

	rev32	v6.16b, v30.16b				//CTR block 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 6

	rev32	v7.16b, v30.16b				//CTR block 7

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 0

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 0

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 1

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 1

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 1

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 1
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 1

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 2
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 1

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 2
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 2

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 2

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 3

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 3
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 3

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 3
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 3

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 4

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 4

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 4
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 5
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 4

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 5
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 5

	sub	x5, x5, #1		//byte_len - 1

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 6

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 6

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 6
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	add	v30.4s, v30.4s, v31.4s		//CTR block 7

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 7

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 7
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 7

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 7
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 7

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 8
	and	x5, x5, #0xffffffffffffff80	//number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 8

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 8

	add	x4, x0, x1, lsr #3		//end_input_ptr
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 9

	ld1	{ v19.16b}, [x3]
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b

	ldp	q27, q28, [x8, #160]				//load rk10, rk11

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 9
	add	x5, x5, x0

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 9
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 9

	cmp	x0, x5				//check if we have <= 8 blocks
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 9

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 9
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 9

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 10
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 10

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 10
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 10

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 10
	ldr	q26, [x8, #192]					//load rk12

	aese	v0.16b, v28.16b						//AES block 0 - round 11
	aese	v1.16b, v28.16b						//AES block 1 - round 11
	aese	v4.16b, v28.16b						//AES block 4 - round 11

	aese	v6.16b, v28.16b						//AES block 6 - round 11
	aese	v5.16b, v28.16b						//AES block 5 - round 11
	aese	v7.16b, v28.16b						//AES block 7 - round 11

	aese	v2.16b, v28.16b						//AES block 2 - round 11
	aese	v3.16b, v28.16b						//AES block 3 - round 11
	b.ge	.L192_dec_tail						//handle tail

	ldp	q8, q9, [x0], #32			//AES block 0, 1 - load ciphertext

	ldp	q10, q11, [x0], #32			//AES block 2, 3 - load ciphertext

	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load ciphertext

.inst	0xce016921	//eor3 v1.16b, v9.16b, v1.16b, v26.16b				//AES block 1 - result
.inst	0xce006900	//eor3 v0.16b, v8.16b, v0.16b, v26.16b				//AES block 0 - result
	stp	q0, q1, [x2], #32			//AES block 0, 1 - store result

	rev32	v0.16b, v30.16b				//CTR block 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8

	rev32	v1.16b, v30.16b				//CTR block 9
	add	v30.4s, v30.4s, v31.4s		//CTR block 9
.inst	0xce036963	//eor3 v3.16b, v11.16b, v3.16b, v26.16b				//AES block 3 - result

.inst	0xce026942	//eor3 v2.16b, v10.16b, v2.16b, v26.16b				//AES block 2 - result
	stp	q2, q3, [x2], #32			//AES block 2, 3 - store result
	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load ciphertext

	rev32	v2.16b, v30.16b				//CTR block 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 10

.inst	0xce046984	//eor3 v4.16b, v12.16b, v4.16b, v26.16b				//AES block 4 - result

	rev32	v3.16b, v30.16b				//CTR block 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 11

.inst	0xce0569a5	//eor3 v5.16b, v13.16b, v5.16b, v26.16b				//AES block 5 - result
	stp	q4, q5, [x2], #32			//AES block 4, 5 - store result
	cmp	x0, x5				//check if we have <= 8 blocks

.inst	0xce0669c6	//eor3 v6.16b, v14.16b, v6.16b, v26.16b				//AES block 6 - result
.inst	0xce0769e7	//eor3 v7.16b, v15.16b, v7.16b, v26.16b				//AES block 7 - result
	rev32	v4.16b, v30.16b				//CTR block 12

	add	v30.4s, v30.4s, v31.4s		//CTR block 12
	stp	q6, q7, [x2], #32			//AES block 6, 7 - store result
	b.ge	.L192_dec_prepretail					//do prepretail

.L192_dec_main_loop:	//main	loop start
	rev64	v9.16b, v9.16b						//GHASH block 8k+1
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0

	rev64	v8.16b, v8.16b						//GHASH block 8k
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v12.16b, v12.16b						//GHASH block 8k+4
	rev64	v11.16b, v11.16b						//GHASH block 8k+3

	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	rev64	v13.16b, v13.16b						//GHASH block 8k+5

	rev32	v7.16b, v30.16b				//CTR block 8k+15
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0

	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1

	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high

	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	ldp	q27, q28, [x8, #64]				//load rk4, rk5

	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	rev64	v15.16b, v15.16b						//GHASH block 8k+7

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5

	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	rev64	v14.16b, v14.16b						//GHASH block 8k+6

	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6

	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low

	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7
	ldp	q28, q26, [x8, #128]				//load rk8, rk9
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7

	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7

.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high

	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
	ldr	d16, [x10]			//MODULO - load modulo constant
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
	rev32	v20.16b, v30.16b					//CTR block 8k+16
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+16

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9
	ldp	q27, q28, [x8, #160]				//load rk10, rk11

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	ldp	q8, q9, [x0], #32			//AES block 8k+8, 8k+9 - load ciphertext

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9
	ldp	q10, q11, [x0], #32			//AES block 8k+10, 8k+11 - load ciphertext

	rev32	v22.16b, v30.16b					//CTR block 8k+17
	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+17

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9
	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10
	ldp	q12, q13, [x0], #32			//AES block 8k+12, 8k+13 - load ciphertext

	rev32	v23.16b, v30.16b					//CTR block 8k+18
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+18
.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10
	ldr	q26, [x8, #192]					//load rk12

	ldp	q14, q15, [x0], #32			//AES block 8k+14, 8k+15 - load ciphertext
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10

	aese	v0.16b, v28.16b						//AES block 8k+8 - round 11
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment
	aese	v1.16b, v28.16b						//AES block 8k+9 - round 11

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10
	aese	v6.16b, v28.16b						//AES block 8k+14 - round 11
	aese	v3.16b, v28.16b						//AES block 8k+11 - round 11

.inst	0xce006900	//eor3 v0.16b, v8.16b, v0.16b, v26.16b				//AES block 8k+8 - result
	rev32	v25.16b, v30.16b					//CTR block 8k+19
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10

	aese	v4.16b, v28.16b						//AES block 8k+12 - round 11
	aese	v2.16b, v28.16b						//AES block 8k+10 - round 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+19

	aese	v7.16b, v28.16b						//AES block 8k+15 - round 11
	aese	v5.16b, v28.16b						//AES block 8k+13 - round 11
	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low

.inst	0xce016921	//eor3 v1.16b, v9.16b, v1.16b, v26.16b				//AES block 8k+9 - result
	stp	q0, q1, [x2], #32			//AES block 8k+8, 8k+9 - store result
.inst	0xce036963	//eor3 v3.16b, v11.16b, v3.16b, v26.16b				//AES block 8k+11 - result

.inst	0xce026942	//eor3 v2.16b, v10.16b, v2.16b, v26.16b				//AES block 8k+10 - result
.inst	0xce0769e7	//eor3 v7.16b, v15.16b, v7.16b, v26.16b				//AES block 8k+15 - result
	stp	q2, q3, [x2], #32			//AES block 8k+10, 8k+11 - store result

.inst	0xce0569a5	//eor3 v5.16b, v13.16b, v5.16b, v26.16b				//AES block 8k+13 - result
.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	mov	v3.16b, v25.16b					//CTR block 8k+19

.inst	0xce046984	//eor3 v4.16b, v12.16b, v4.16b, v26.16b				//AES block 8k+12 - result
	stp	q4, q5, [x2], #32			//AES block 8k+12, 8k+13 - store result
	cmp	x0, x5				//.LOOP CONTROL

.inst	0xce0669c6	//eor3 v6.16b, v14.16b, v6.16b, v26.16b				//AES block 8k+14 - result
	stp	q6, q7, [x2], #32			//AES block 8k+14, 8k+15 - store result
	mov	v0.16b, v20.16b					//CTR block 8k+16

	mov	v1.16b, v22.16b					//CTR block 8k+17
	mov	v2.16b, v23.16b					//CTR block 8k+18

	rev32	v4.16b, v30.16b				//CTR block 8k+20
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+20
	b.lt	.L192_dec_main_loop

.L192_dec_prepretail:	//PREPRETAIL
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v8.16b, v8.16b						//GHASH block 8k
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0

	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	rev64	v9.16b, v9.16b						//GHASH block 8k+1

	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	rev32	v7.16b, v30.16b				//CTR block 8k+15

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1

	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1

	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1

	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2
	rev64	v13.16b, v13.16b						//GHASH block 8k+5
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low

.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2

	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3

	rev64	v15.16b, v15.16b						//GHASH block 8k+7

.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	rev64	v12.16b, v12.16b						//GHASH block 8k+4

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4

	rev64	v14.16b, v14.16b						//GHASH block 8k+6
	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5

	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5

	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high

	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5

	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6

	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6

	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high

.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7

	ldp	q28, q26, [x8, #128]				//load rk8, rk9
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7

	ldr	d16, [x10]			//MODULO - load modulo constant
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
	ldp	q27, q28, [x8, #160]				//load rk10, rk11

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ldr	q26, [x8, #192]					//load rk12
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10

	aese	v0.16b, v28.16b						//AES block 8k+8 - round 11
.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	aese	v5.16b, v28.16b						//AES block 8k+13 - round 11

	aese	v2.16b, v28.16b						//AES block 8k+10 - round 11
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10

	aese	v6.16b, v28.16b						//AES block 8k+14 - round 11
	aese	v4.16b, v28.16b						//AES block 8k+12 - round 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	aese	v3.16b, v28.16b						//AES block 8k+11 - round 11
	aese	v1.16b, v28.16b						//AES block 8k+9 - round 11
	aese	v7.16b, v28.16b						//AES block 8k+15 - round 11

.L192_dec_tail:	//TAIL

	sub	x5, x4, x0 	//main_end_input_ptr is number of bytes left to process

	ldp	q20, q21, [x3, #128]			//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q9, [x0], #16				//AES block 8k+8 - load ciphertext

	ldp	q24, q25, [x3, #192]			//load h8k | h7k
	ext	v25.16b, v25.16b, v25.16b, #8

	mov	v29.16b, v26.16b

	ldp	q22, q23, [x3, #160]			//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	ext	v23.16b, v23.16b, v23.16b, #8
	ext	v16.16b, v19.16b, v19.16b, #8				//prepare final partial tag

.inst	0xce00752c	//eor3 v12.16b, v9.16b, v0.16b, v29.16b				//AES block 8k+8 - result
	cmp	x5, #112
	b.gt	.L192_dec_blocks_more_than_7

	mov	v7.16b, v6.16b
	movi	v17.8b, #0
	sub	v30.4s, v30.4s, v31.4s

	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b
	mov	v4.16b, v3.16b

	cmp	x5, #96
	movi	v19.8b, #0
	mov	v3.16b, v2.16b

	mov	v2.16b, v1.16b
	movi	v18.8b, #0
	b.gt	.L192_dec_blocks_more_than_6

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	mov	v4.16b, v3.16b
	mov	v3.16b, v1.16b

	sub	v30.4s, v30.4s, v31.4s
	cmp	x5, #80
	b.gt	.L192_dec_blocks_more_than_5

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b

	mov	v5.16b, v4.16b
	mov	v4.16b, v1.16b
	cmp	x5, #64

	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L192_dec_blocks_more_than_4

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b

	mov	v5.16b, v1.16b
	cmp	x5, #48
	b.gt	.L192_dec_blocks_more_than_3

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b
	cmp	x5, #32

	mov	v6.16b, v1.16b
	ldr	q24, [x3, #96]				//load h4k | h3k
	b.gt	.L192_dec_blocks_more_than_2

	sub	v30.4s, v30.4s, v31.4s

	mov	v7.16b, v1.16b
	cmp	x5, #16
	b.gt	.L192_dec_blocks_more_than_1

	sub	v30.4s, v30.4s, v31.4s
	ldr	q21, [x3, #48]				//load h2k | h1k
	b	.L192_dec_blocks_less_than_1
.L192_dec_blocks_more_than_7:	//blocks	left >  7
	rev64	v8.16b, v9.16b						//GHASH final-7 block

	ins	v18.d[0], v24.d[1]					//GHASH final-7 block - mid
	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH final-7 block - high
	ins	v27.d[0], v8.d[1]					//GHASH final-7 block - mid
	ldr	q9, [x0], #16				//AES final-6 block - load ciphertext

	pmull	v19.1q, v8.1d, v25.1d				//GHASH final-7 block - low

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-7 block - mid
	st1	{ v12.16b}, [x2], #16			 	//AES final-7 block  - store result

.inst	0xce01752c	//eor3 v12.16b, v9.16b, v1.16b, v29.16b				//AES final-6 block - result

	pmull	v18.1q, v27.1d, v18.1d			 	//GHASH final-7 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
.L192_dec_blocks_more_than_6:	//blocks	left >  6

	rev64	v8.16b, v9.16b						//GHASH final-6 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ldr	q9, [x0], #16				//AES final-5 block - load ciphertext
	ins	v27.d[0], v8.d[1]					//GHASH final-6 block - mid

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-6 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-6 block - high

	st1	{ v12.16b}, [x2], #16			 	//AES final-6 block - store result
.inst	0xce02752c	//eor3 v12.16b, v9.16b, v2.16b, v29.16b				//AES final-5 block - result

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-6 block - high
	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-6 block - mid
	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-6 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-6 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-6 block - low
.L192_dec_blocks_more_than_5:	//blocks	left >  5

	rev64	v8.16b, v9.16b						//GHASH final-5 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-5 block - mid

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-5 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-5 block - mid
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-5 block - high

	ldr	q9, [x0], #16				//AES final-4 block - load ciphertext

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-5 block - high
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-5 block - low

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-5 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-5 block - low
	movi	v16.8b, #0						//supress further partial tag feed in
	st1	{ v12.16b}, [x2], #16			 	//AES final-5 block - store result

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-5 block - mid
.inst	0xce03752c	//eor3 v12.16b, v9.16b, v3.16b, v29.16b				//AES final-4 block - result
.L192_dec_blocks_more_than_4:	//blocks	left >  4

	rev64	v8.16b, v9.16b						//GHASH final-4 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	movi	v16.8b, #0						//supress further partial tag feed in

	ldr	q9, [x0], #16				//AES final-3 block - load ciphertext
	ins	v27.d[0], v8.d[1]					//GHASH final-4 block - mid
	pmull	v26.1q, v8.1d, v20.1d				//GHASH final-4 block - low

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-4 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-4 block - low

	pmull	v27.1q, v27.1d, v21.1d				//GHASH final-4 block - mid
	st1	{ v12.16b}, [x2], #16			 	//AES final-4 block - store result
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final-4 block - high

.inst	0xce04752c	//eor3 v12.16b, v9.16b, v4.16b, v29.16b				//AES final-3 block - result

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-4 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-4 block - high
.L192_dec_blocks_more_than_3:	//blocks	left >  3

	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v8.16b, v9.16b						//GHASH final-3 block
	ldr	q9, [x0], #16				//AES final-2 block - load ciphertext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-3 block - mid
	pmull2	v28.1q, v8.2d, v25.2d				//GHASH final-3 block - high

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-3 block - high
	movi	v16.8b, #0						//supress further partial tag feed in
	pmull	v26.1q, v8.1d, v25.1d				//GHASH final-3 block - low

	st1	{ v12.16b}, [x2], #16			 	//AES final-3 block - store result
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-3 block - mid
.inst	0xce05752c	//eor3 v12.16b, v9.16b, v5.16b, v29.16b				//AES final-2 block - result

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-3 block - low
	ldr	q24, [x3, #96]				//load h4k | h3k

	ins	v27.d[1], v27.d[0]					//GHASH final-3 block - mid

	pmull2	v27.1q, v27.2d, v24.2d				//GHASH final-3 block - mid

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-3 block - mid
.L192_dec_blocks_more_than_2:	//blocks	left >  2

	rev64	v8.16b, v9.16b						//GHASH final-2 block
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-2 block - mid
	ldr	q9, [x0], #16				//AES final-1 block - load ciphertext

	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-2 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-2 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-2 block - high
	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-2 block - low

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-2 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-2 block - low
	st1	{ v12.16b}, [x2], #16			 	//AES final-2 block - store result

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-2 block - mid
.inst	0xce06752c	//eor3 v12.16b, v9.16b, v6.16b, v29.16b				//AES final-1 block - result
.L192_dec_blocks_more_than_1:	//blocks	left >  1

	rev64	v8.16b, v9.16b						//GHASH final-1 block
	ldr	q9, [x0], #16				//AES final block - load ciphertext
	ldr	q22, [x3, #64]				//load h1l | h1h
	ext	v22.16b, v22.16b, v22.16b, #8

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	movi	v16.8b, #0						//supress further partial tag feed in
	ldr	q21, [x3, #48]				//load h2k | h1k

	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-1 block - low
	ins	v27.d[0], v8.d[1]					//GHASH final-1 block - mid
	st1	{ v12.16b}, [x2], #16			 	//AES final-1 block - store result

	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-1 block - high

.inst	0xce07752c	//eor3 v12.16b, v9.16b, v7.16b, v29.16b				//AES final block - result

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-1 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-1 block - mid

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-1 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-1 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-1 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-1 block - high
.L192_dec_blocks_less_than_1:	//blocks	left <= 1

	rev32	v30.16b, v30.16b
	and	x1, x1, #127				//bit_length %= 128

	sub	x1, x1, #128				//bit_length -= 128
	str	q30, [x16]					//store the updated counter

	neg	x1, x1				//bit_length = 128 - #bits in input (in range [1,128])
	mvn	x6, xzr						//temp0_x = 0xffffffffffffffff

	and	x1, x1, #127				//bit_length %= 128

	mvn	x7, xzr						//temp1_x = 0xffffffffffffffff
	lsr	x6, x6, x1				//temp0_x is mask for top 64b of last block
	cmp	x1, #64

	csel	x13, x7, x6, lt
	csel	x14, x6, xzr, lt
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8

	mov	v0.d[1], x14
	ld1	{ v26.16b}, [x2]					//load existing bytes where the possibly partial last block is to be stored

	mov	v0.d[0], x13					//ctr0b is mask for last block

	and	v9.16b, v9.16b, v0.16b					//possibly partial last block has zeroes in highest bits
	bif	v12.16b, v26.16b, v0.16b					//insert existing bytes in top end of result before storing

	rev64	v8.16b, v9.16b						//GHASH final block

	st1	{ v12.16b}, [x2]				//store all 16B

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v16.d[0], v8.d[1]					//GHASH final block - mid
	pmull	v26.1q, v8.1d, v20.1d				//GHASH final block - low

	eor	v16.8b, v16.8b, v8.8b				//GHASH final block - mid
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final block - high
	eor	v19.16b, v19.16b, v26.16b					//GHASH final block - low

	pmull	v16.1q, v16.1d, v21.1d				//GHASH final block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final block - high

	eor	v14.16b, v17.16b, v19.16b				//MODULO - karatsuba tidy up
	eor	v18.16b, v18.16b, v16.16b				//GHASH final block - mid
	ldr	d16, [x10]			//MODULO - load modulo constant

	pmull	v21.1q, v17.1d, v16.1d			//MODULO - top 64b align with mid
	ext	v17.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment

	eor	v18.16b, v18.16b, v14.16b				//MODULO - karatsuba tidy up

.inst	0xce115652	//eor3 v18.16b, v18.16b, v17.16b, v21.16b			//MODULO - fold into mid

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ext	v18.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment

.inst	0xce124673	//eor3 v19.16b, v19.16b, v18.16b, v17.16b			//MODULO - fold into low
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	st1	{ v19.16b }, [x3]

	mov	x0, x9

	ldp	d10, d11, [sp, #16]
	ldp	d12, d13, [sp, #32]
	ldp	d14, d15, [sp, #48]
	ldp	d8, d9, [sp], #80
	ret

.L192_dec_ret:
	mov	w0, #0x0
	ret
.size	unroll8_eor3_aes_gcm_dec_192_kernel,.-unroll8_eor3_aes_gcm_dec_192_kernel
.globl	unroll8_eor3_aes_gcm_enc_256_kernel
.type	unroll8_eor3_aes_gcm_enc_256_kernel,%function
.align	4
unroll8_eor3_aes_gcm_enc_256_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L256_enc_ret
	stp	d8, d9, [sp, #-80]!
	lsr	x9, x1, #3
	mov	x16, x4
	mov	x8, x5
	stp	d10, d11, [sp, #16]
	stp	d12, d13, [sp, #32]
	stp	d14, d15, [sp, #48]
	mov	x5, #0xc200000000000000
	stp	x5, xzr, [sp, #64]
	add	x10, sp, #64

	ld1	{ v0.16b}, [x16]					//CTR block 0

	mov	x5, x9

	mov	x15, #0x100000000			//set up counter increment
	movi	v31.16b, #0x0
	mov	v31.d[1], x15
	sub	x5, x5, #1		//byte_len - 1

	and	x5, x5, #0xffffffffffffff80	//number of bytes to be processed in main loop (at least 1 byte must be handled by tail)

	add	x5, x5, x0

	rev32	v30.16b, v0.16b				//set up reversed counter

	add	v30.4s, v30.4s, v31.4s		//CTR block 0

	rev32	v1.16b, v30.16b				//CTR block 1
	add	v30.4s, v30.4s, v31.4s		//CTR block 1

	rev32	v2.16b, v30.16b				//CTR block 2
	add	v30.4s, v30.4s, v31.4s		//CTR block 2

	rev32	v3.16b, v30.16b				//CTR block 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 3

	rev32	v4.16b, v30.16b				//CTR block 4
	add	v30.4s, v30.4s, v31.4s		//CTR block 4

	rev32	v5.16b, v30.16b				//CTR block 5
	add	v30.4s, v30.4s, v31.4s		//CTR block 5
	ldp	q26, q27, [x8, #0]				 	//load rk0, rk1

	rev32	v6.16b, v30.16b				//CTR block 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 6

	rev32	v7.16b, v30.16b				//CTR block 7

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 0

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 0

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 1

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 1

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 1

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 1

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 2
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 1

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 2
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 2

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 2
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 2

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 3
	ldp	q27, q28, [x8, #64]				//load rk4, rk5

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 3

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 3
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 3

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 3
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 3

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 4

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 4

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 4

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 5

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 5

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 6
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 6

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 6
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 7

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 7

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 7

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 7

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 8

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 8

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 8

	ld1	{ v19.16b}, [x3]
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	ldp	q27, q28, [x8, #160]				//load rk10, rk11

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 9
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 9

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 9
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 9
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 9

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 9

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 10
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 10
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 9

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 10
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 10

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 10
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 10

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 11
	ldp	q26, q27, [x8, #192]				//load rk12, rk13
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 11

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 11
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 11
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 11

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 11
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 11
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 11

	add	v30.4s, v30.4s, v31.4s		//CTR block 7
	ldr	q28, [x8, #224]					//load rk14

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 12
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 12
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 12

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 12
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 12
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 12

	aese	v2.16b, v27.16b						//AES block 2 - round 13
	aese	v1.16b, v27.16b						//AES block 1 - round 13
	aese	v4.16b, v27.16b						//AES block 4 - round 13

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 12
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 12

	aese	v0.16b, v27.16b						//AES block 0 - round 13
	aese	v5.16b, v27.16b						//AES block 5 - round 13

	aese	v6.16b, v27.16b						//AES block 6 - round 13
	aese	v7.16b, v27.16b						//AES block 7 - round 13
	aese	v3.16b, v27.16b						//AES block 3 - round 13

	add	x4, x0, x1, lsr #3		//end_input_ptr
	cmp	x0, x5				//check if we have <= 8 blocks
	b.ge	.L256_enc_tail						//handle tail

	ldp	q8, q9, [x0], #32			//AES block 0, 1 - load plaintext

	ldp	q10, q11, [x0], #32			//AES block 2, 3 - load plaintext

.inst	0xce007108	//eor3 v8.16b, v8.16b, v0.16b, v28.16b				//AES block 0 - result
	rev32	v0.16b, v30.16b				//CTR block 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8

.inst	0xce017129	//eor3 v9.16b, v9.16b, v1.16b, v28.16b				//AES block 1 - result
.inst	0xce03716b	//eor3 v11.16b, v11.16b, v3.16b, v28.16b				//AES block 3 - result

	rev32	v1.16b, v30.16b				//CTR block 9
	add	v30.4s, v30.4s, v31.4s		//CTR block 9
	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load plaintext

	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load plaintext
.inst	0xce02714a	//eor3 v10.16b, v10.16b, v2.16b, v28.16b				//AES block 2 - result
	cmp	x0, x5				//check if we have <= 8 blocks

	rev32	v2.16b, v30.16b				//CTR block 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 10
	stp	q8, q9, [x2], #32			//AES block 0, 1 - store result

	stp	q10, q11, [x2], #32			//AES block 2, 3 - store result

	rev32	v3.16b, v30.16b				//CTR block 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 11

.inst	0xce04718c	//eor3 v12.16b, v12.16b, v4.16b, v28.16b				//AES block 4 - result

.inst	0xce0771ef	//eor3 v15.16b, v15.16b, v7.16b, v28.16b				//AES block 7 - result
.inst	0xce0671ce	//eor3 v14.16b, v14.16b, v6.16b, v28.16b				//AES block 6 - result
.inst	0xce0571ad	//eor3 v13.16b, v13.16b, v5.16b, v28.16b				//AES block 5 - result

	stp	q12, q13, [x2], #32			//AES block 4, 5 - store result
	rev32	v4.16b, v30.16b				//CTR block 12

	stp	q14, q15, [x2], #32			//AES block 6, 7 - store result
	add	v30.4s, v30.4s, v31.4s		//CTR block 12
	b.ge	.L256_enc_prepretail					//do prepretail

.L256_enc_main_loop:	//main	loop start
	ldp	q26, q27, [x8, #0]					//load rk0, rk1

	rev32	v5.16b, v30.16b				//CTR block 8k+13
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k

	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	rev64	v9.16b, v9.16b						//GHASH block 8k+1

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14
	rev64	v8.16b, v8.16b						//GHASH block 8k

	rev64	v12.16b, v12.16b						//GHASH block 8k+4
	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	rev32	v7.16b, v30.16b				//CTR block 8k+15

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	eor	v8.16b, v8.16b, v19.16b				 	//PRE 1
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high

	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	rev64	v14.16b, v14.16b						//GHASH block 8k+6
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	rev64	v10.16b, v10.16b						//GHASH block 8k+2

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3

	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	rev64	v13.16b, v13.16b						//GHASH block 8k+5

	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high
	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4

	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4

	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5

	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	rev64	v15.16b, v15.16b						//GHASH block 8k+7

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5

	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5

	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6

	ldp	q28, q26, [x8, #128]				//load rk8, rk9
	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid

	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low

	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7

	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8

	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8

.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9

	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9

	ldp	q27, q28, [x8, #160]				//load rk10, rk11
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	ldr	d16, [x10]			//MODULO - load modulo constant
	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9

.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high

	ldp	q26, q27, [x8, #192]				//load rk12, rk13
	rev32	v20.16b, v30.16b					//CTR block 8k+16

	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment
	ldp	q8, q9, [x0], #32			//AES block 8k+8, 8k+9 - load plaintext
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 11

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+16
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 11

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 11
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 11

	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 11

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 12
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 11

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 12
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 12
	rev32	v22.16b, v30.16b					//CTR block 8k+17

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+17
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 11
.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 12
	ldr	q28, [x8, #224]					//load rk14
	aese	v7.16b, v27.16b						//AES block 8k+15 - round 13

	ldp	q10, q11, [x0], #32			//AES block 8k+10, 8k+11 - load plaintext
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 12
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 12

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 12
	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load plaintext

	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load plaintext
	aese	v2.16b, v27.16b						//AES block 8k+10 - round 13
	aese	v4.16b, v27.16b						//AES block 8k+12 - round 13

	rev32	v23.16b, v30.16b					//CTR block 8k+18
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+18
	aese	v5.16b, v27.16b						//AES block 8k+13 - round 13

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 12
	aese	v3.16b, v27.16b						//AES block 8k+11 - round 13
	cmp	x0, x5				//.LOOP CONTROL

.inst	0xce02714a	//eor3 v10.16b, v10.16b, v2.16b, v28.16b				//AES block 8k+10 - result
	rev32	v25.16b, v30.16b					//CTR block 8k+19
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+19

	aese	v0.16b, v27.16b						//AES block 8k+8 - round 13
	aese	v6.16b, v27.16b						//AES block 8k+14 - round 13
.inst	0xce0571ad	//eor3 v13.16b, v13.16b, v5.16b, v28.16b				//AES block 5 - result

	ext	v21.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment
	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v1.16b, v27.16b						//AES block 8k+9 - round 13

.inst	0xce04718c	//eor3 v12.16b, v12.16b, v4.16b, v28.16b				//AES block 4 - result
	rev32	v4.16b, v30.16b				//CTR block 8k+20
.inst	0xce03716b	//eor3 v11.16b, v11.16b, v3.16b, v28.16b				//AES block 8k+11 - result

	mov	v3.16b, v25.16b					//CTR block 8k+19
.inst	0xce017129	//eor3 v9.16b, v9.16b, v1.16b, v28.16b				//AES block 8k+9 - result
.inst	0xce007108	//eor3 v8.16b, v8.16b, v0.16b, v28.16b				//AES block 8k+8 - result

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+20
	stp	q8, q9, [x2], #32			//AES block 8k+8, 8k+9 - store result
	mov	v2.16b, v23.16b					//CTR block 8k+18

.inst	0xce0771ef	//eor3 v15.16b, v15.16b, v7.16b, v28.16b				//AES block 7 - result
.inst	0xce154673	//eor3 v19.16b, v19.16b, v21.16b, v17.16b		 	//MODULO - fold into low
	stp	q10, q11, [x2], #32			//AES block 8k+10, 8k+11 - store result

.inst	0xce0671ce	//eor3 v14.16b, v14.16b, v6.16b, v28.16b				//AES block 6 - result
	mov	v1.16b, v22.16b					//CTR block 8k+17
	stp	q12, q13, [x2], #32			//AES block 4, 5 - store result

	stp	q14, q15, [x2], #32			//AES block 6, 7 - store result
	mov	v0.16b, v20.16b					//CTR block 8k+16
	b.lt	.L256_enc_main_loop

.L256_enc_prepretail:	//PREPRETAIL
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	ldp	q26, q27, [x8, #0]					//load rk0, rk1
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	rev64	v10.16b, v10.16b						//GHASH block 8k+2

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	rev64	v13.16b, v13.16b						//GHASH block 8k+5
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k

	rev32	v7.16b, v30.16b				//CTR block 8k+15

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0

	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0
	rev64	v8.16b, v8.16b						//GHASH block 8k
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1

	rev64	v9.16b, v9.16b						//GHASH block 8k+1
	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1

	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1

	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	eor	v8.16b, v8.16b, v19.16b					//PRE 1

	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	rev64	v14.16b, v14.16b						//GHASH block 8k+6
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high

	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3

	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5
	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4

	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid

	rev64	v12.16b, v12.16b						//GHASH block 8k+4
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	rev64	v15.16b, v15.16b						//GHASH block 8k+7
	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6

	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6

	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8

	ldp	q28, q26, [x8, #128]				//load rk8, rk9
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7

	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7

	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8

	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high
	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid

	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low
.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high

	ldp	q27, q28, [x8, #160]				//load rk10, rk11
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
	ldr	d16, [x10]			//MODULO - load modulo constant

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10

	pmull	v29.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 11

	ldp	q26, q27, [x8, #192]				//load rk12, rk13
	ext	v21.16b, v17.16b, v17.16b, #8			 	//MODULO - other top alignment
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 11

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 11
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 11

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 11
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 11
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 11

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 11
	ldr	q28, [x8, #224]					//load rk14

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 12
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 12
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 12

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 12
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 12
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 12
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 12
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 12
	aese	v0.16b, v27.16b						//AES block 8k+8 - round 13

.inst	0xce154673	//eor3 v19.16b, v19.16b, v21.16b, v17.16b		 	//MODULO - fold into low
	aese	v5.16b, v27.16b						//AES block 8k+13 - round 13
	aese	v1.16b, v27.16b						//AES block 8k+9 - round 13

	aese	v3.16b, v27.16b						//AES block 8k+11 - round 13
	aese	v4.16b, v27.16b						//AES block 8k+12 - round 13
	aese	v7.16b, v27.16b						//AES block 8k+15 - round 13

	aese	v2.16b, v27.16b						//AES block 8k+10 - round 13
	aese	v6.16b, v27.16b						//AES block 8k+14 - round 13
.L256_enc_tail:	//TAIL

	ldp	q24, q25, [x3, #192]			//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	sub	x5, x4, x0		//main_end_input_ptr is number of bytes left to process

	ldr	q8, [x0], #16				//AES block 8k+8 - load plaintext

	ldp	q20, q21, [x3, #128]			//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8

	ext	v16.16b, v19.16b, v19.16b, #8				//prepare final partial tag
	ldp	q22, q23, [x3, #160]			//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	ext	v23.16b, v23.16b, v23.16b, #8
	mov	v29.16b, v28.16b

	cmp	x5, #112
.inst	0xce007509	//eor3 v9.16b, v8.16b, v0.16b, v29.16b				//AES block 8k+8 - result
	b.gt	.L256_enc_blocks_more_than_7

	movi	v19.8b, #0
	mov	v7.16b, v6.16b
	movi	v17.8b, #0

	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b
	mov	v4.16b, v3.16b

	mov	v3.16b, v2.16b
	sub	v30.4s, v30.4s, v31.4s
	mov	v2.16b, v1.16b

	movi	v18.8b, #0
	cmp	x5, #96
	b.gt	.L256_enc_blocks_more_than_6

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b
	cmp	x5, #80

	mov	v5.16b, v4.16b
	mov	v4.16b, v3.16b
	mov	v3.16b, v1.16b

	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L256_enc_blocks_more_than_5

	mov	v7.16b, v6.16b
	sub	v30.4s, v30.4s, v31.4s

	mov	v6.16b, v5.16b
	mov	v5.16b, v4.16b

	cmp	x5, #64
	mov	v4.16b, v1.16b
	b.gt	.L256_enc_blocks_more_than_4

	cmp	x5, #48
	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b

	mov	v5.16b, v1.16b
	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L256_enc_blocks_more_than_3

	cmp	x5, #32
	mov	v7.16b, v6.16b
	ldr	q24, [x3, #96]				//load h4k | h3k

	mov	v6.16b, v1.16b
	sub	v30.4s, v30.4s, v31.4s
	b.gt	.L256_enc_blocks_more_than_2

	mov	v7.16b, v1.16b

	sub	v30.4s, v30.4s, v31.4s
	cmp	x5, #16
	b.gt	.L256_enc_blocks_more_than_1

	sub	v30.4s, v30.4s, v31.4s
	ldr	q21, [x3, #48]				//load h2k | h1k
	b	.L256_enc_blocks_less_than_1
.L256_enc_blocks_more_than_7:	//blocks	left >  7
	st1	{ v9.16b}, [x2], #16				//AES final-7 block  - store result

	rev64	v8.16b, v9.16b						//GHASH final-7 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ldr	q9, [x0], #16				//AES final-6 block - load plaintext

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH final-7 block - high
	ins	v27.d[0], v8.d[1]					//GHASH final-7 block - mid
	ins	v18.d[0], v24.d[1]					//GHASH final-7 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-7 block - mid
.inst	0xce017529	//eor3 v9.16b, v9.16b, v1.16b, v29.16b			//AES final-6 block - result

	pmull	v18.1q, v27.1d, v18.1d				//GHASH final-7 block - mid
	pmull	v19.1q, v8.1d, v25.1d				//GHASH final-7 block - low
.L256_enc_blocks_more_than_6:	//blocks	left >  6

	st1	{ v9.16b}, [x2], #16				//AES final-6 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-6 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-6 block - low
	ins	v27.d[0], v8.d[1]					//GHASH final-6 block - mid
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-6 block - high

	ldr	q9, [x0], #16				//AES final-5 block - load plaintext

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-6 block - low

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-6 block - mid

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-6 block - mid
.inst	0xce027529	//eor3 v9.16b, v9.16b, v2.16b, v29.16b			//AES final-5 block - result

	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-6 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-6 block - high
.L256_enc_blocks_more_than_5:	//blocks	left >  5

	st1	{ v9.16b}, [x2], #16				//AES final-5 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-5 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-5 block - mid

	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-5 block - high

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-5 block - high
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-5 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-5 block - mid

	ldr	q9, [x0], #16				//AES final-4 block - load plaintext
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-5 block - low

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-5 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-5 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-5 block - mid
.inst	0xce037529	//eor3 v9.16b, v9.16b, v3.16b, v29.16b			//AES final-4 block - result
.L256_enc_blocks_more_than_4:	//blocks	left >  4

	st1	{ v9.16b}, [x2], #16				//AES final-4 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-4 block

	ldr	q9, [x0], #16				//AES final-3 block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-4 block - mid
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final-4 block - high

.inst	0xce047529	//eor3 v9.16b, v9.16b, v4.16b, v29.16b			//AES final-3 block - result
	pmull	v26.1q, v8.1d, v20.1d				//GHASH final-4 block - low

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-4 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-4 block - low

	pmull	v27.1q, v27.1d, v21.1d				//GHASH final-4 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-4 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-4 block - high
.L256_enc_blocks_more_than_3:	//blocks	left >  3

	st1	{ v9.16b}, [x2], #16				//AES final-3 block - store result

	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v8.16b, v9.16b						//GHASH final-3 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-3 block - mid
	pmull2	v28.1q, v8.2d, v25.2d				//GHASH final-3 block - high

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-3 block - high
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-3 block - mid
	ldr	q24, [x3, #96]				//load h4k | h3k

	ins	v27.d[1], v27.d[0]					//GHASH final-3 block - mid
	ldr	q9, [x0], #16				//AES final-2 block - load plaintext

	pmull2	v27.1q, v27.2d, v24.2d				//GHASH final-3 block - mid
	pmull	v26.1q, v8.1d, v25.1d				//GHASH final-3 block - low

.inst	0xce057529	//eor3 v9.16b, v9.16b, v5.16b, v29.16b			//AES final-2 block - result
	movi	v16.8b, #0						//supress further partial tag feed in

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-3 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-3 block - low
.L256_enc_blocks_more_than_2:	//blocks	left >  2

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8

	st1	{ v9.16b}, [x2], #16			 	//AES final-2 block - store result

	rev64	v8.16b, v9.16b						//GHASH final-2 block
	ldr	q9, [x0], #16				//AES final-1 block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-2 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in

	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-2 block - high
.inst	0xce067529	//eor3 v9.16b, v9.16b, v6.16b, v29.16b			//AES final-1 block - result

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-2 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-2 block - high

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-2 block - mid
	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-2 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-2 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-2 block - low
.L256_enc_blocks_more_than_1:	//blocks	left >  1

	st1	{ v9.16b}, [x2], #16				//AES final-1 block - store result

	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	rev64	v8.16b, v9.16b						//GHASH final-1 block
	ldr	q9, [x0], #16				//AES final block - load plaintext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	movi	v16.8b, #0						//supress further partial tag feed in

	ins	v27.d[0], v8.d[1]					//GHASH final-1 block - mid
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-1 block - high

.inst	0xce077529	//eor3 v9.16b, v9.16b, v7.16b, v29.16b			//AES final block - result
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-1 block - high

	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-1 block - low
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-1 block - mid

	ldr	q21, [x3, #48]				//load h2k | h1k

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-1 block - low
	ins	v27.d[1], v27.d[0]					//GHASH final-1 block - mid

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-1 block - mid

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-1 block - mid
.L256_enc_blocks_less_than_1:	//blocks	left <= 1

	and	x1, x1, #127				//bit_length %= 128

	sub	x1, x1, #128				//bit_length -= 128

	neg	x1, x1				//bit_length = 128 - #bits in input (in range [1,128])

	mvn	x6, xzr						//temp0_x = 0xffffffffffffffff
	and	x1, x1, #127				//bit_length %= 128

	lsr	x6, x6, x1				//temp0_x is mask for top 64b of last block
	cmp	x1, #64
	mvn	x7, xzr						//temp1_x = 0xffffffffffffffff

	csel	x14, x6, xzr, lt
	csel	x13, x7, x6, lt

	mov	v0.d[0], x13					//ctr0b is mask for last block
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8

	ld1	{ v26.16b}, [x2]					//load existing bytes where the possibly partial last block is to be stored
	mov	v0.d[1], x14

	and	v9.16b, v9.16b, v0.16b					//possibly partial last block has zeroes in highest bits

	rev64	v8.16b, v9.16b						//GHASH final block

	rev32	v30.16b, v30.16b
	bif	v9.16b, v26.16b, v0.16b					//insert existing bytes in top end of result before storing
	str	q30, [x16]					//store the updated counter

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	st1	{ v9.16b}, [x2]				//store all 16B

	ins	v16.d[0], v8.d[1]					//GHASH final block - mid
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final block - high
	pmull	v26.1q, v8.1d, v20.1d				//GHASH final block - low

	eor	v17.16b, v17.16b, v28.16b					//GHASH final block - high
	eor	v19.16b, v19.16b, v26.16b					//GHASH final block - low

	eor	v16.8b, v16.8b, v8.8b				//GHASH final block - mid

	pmull	v16.1q, v16.1d, v21.1d				//GHASH final block - mid

	eor	v18.16b, v18.16b, v16.16b				//GHASH final block - mid
	ldr	d16, [x10]			//MODULO - load modulo constant

	ext	v21.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	pmull	v29.1q, v17.1d, v16.1d			//MODULO - top 64b align with mid

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	ext	v21.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment

.inst	0xce115673	//eor3 v19.16b, v19.16b, v17.16b, v21.16b		 	//MODULO - fold into low
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	st1	{ v19.16b }, [x3]
	mov	x0, x9					//return sizes

	ldp	d10, d11, [sp, #16]
	ldp	d12, d13, [sp, #32]
	ldp	d14, d15, [sp, #48]
	ldp	d8, d9, [sp], #80
	ret

.L256_enc_ret:
	mov	w0, #0x0
	ret
.size	unroll8_eor3_aes_gcm_enc_256_kernel,.-unroll8_eor3_aes_gcm_enc_256_kernel
.globl	unroll8_eor3_aes_gcm_dec_256_kernel
.type	unroll8_eor3_aes_gcm_dec_256_kernel,%function
.align	4
unroll8_eor3_aes_gcm_dec_256_kernel:
	AARCH64_VALID_CALL_TARGET
	cbz	x1, .L256_dec_ret
	stp	d8, d9, [sp, #-80]!
	lsr	x9, x1, #3
	mov	x16, x4
	mov	x8, x5
	stp	d10, d11, [sp, #16]
	stp	d12, d13, [sp, #32]
	stp	d14, d15, [sp, #48]
	mov	x5, #0xc200000000000000
	stp	x5, xzr, [sp, #64]
	add	x10, sp, #64

	ld1	{ v0.16b}, [x16]					//CTR block 0

	mov	x15, #0x100000000			//set up counter increment
	movi	v31.16b, #0x0
	mov	v31.d[1], x15
	mov	x5, x9

	sub	x5, x5, #1		//byte_len - 1

	rev32	v30.16b, v0.16b				//set up reversed counter

	add	v30.4s, v30.4s, v31.4s		//CTR block 0

	rev32	v1.16b, v30.16b				//CTR block 1
	add	v30.4s, v30.4s, v31.4s		//CTR block 1

	rev32	v2.16b, v30.16b				//CTR block 2
	add	v30.4s, v30.4s, v31.4s		//CTR block 2
	ldp	q26, q27, [x8, #0]				  	//load rk0, rk1

	rev32	v3.16b, v30.16b				//CTR block 3
	add	v30.4s, v30.4s, v31.4s		//CTR block 3

	rev32	v4.16b, v30.16b				//CTR block 4
	add	v30.4s, v30.4s, v31.4s		//CTR block 4

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 0

	rev32	v5.16b, v30.16b				//CTR block 5
	add	v30.4s, v30.4s, v31.4s		//CTR block 5

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 0

	rev32	v6.16b, v30.16b				//CTR block 6
	add	v30.4s, v30.4s, v31.4s		//CTR block 6

	rev32	v7.16b, v30.16b				//CTR block 7
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 0

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b		        //AES block 6 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 0
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b		        //AES block 7 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b		        //AES block 6 - round 1
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b		        //AES block 4 - round 1
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b		        //AES block 0 - round 1

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 1
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 1

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 1
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 1

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 2

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 2
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 2

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 2
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 2
	ldp	q27, q28, [x8, #64]				//load rk4, rk5

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 3
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 3

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 3
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 3

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 3
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 3
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 3

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 3

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 4
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 4

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 4
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 4

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 4
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 4

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 5

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 5

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 5

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 5
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 5

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 5

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 6

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 6
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 6
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 7
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 7

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 7
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 7
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 7

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 7
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 7

	and	x5, x5, #0xffffffffffffff80 //number of bytes to be processed in main loop (at least 1 byte must be handled by tail)
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 8

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 8
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 8
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 8

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 8
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 8

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 9

	ld1	{ v19.16b}, [x3]
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	ldp	q27, q28, [x8, #160]				//load rk10, rk11
	add	x4, x0, x1, lsr #3 //end_input_ptr
	add	x5, x5, x0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 9
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 9

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 9
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 9

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 9

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 9

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 10

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 10
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 10

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 10
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 10
	ldp	q26, q27, [x8, #192]				//load rk12, rk13

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 11
	add	v30.4s, v30.4s, v31.4s //CTR block 7

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 11
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 11
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 11

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 11
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 11
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 11

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 11
	ldr	q28, [x8, #224]					//load rk14

	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 1 - round 12
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 4 - round 12
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 5 - round 12

	cmp	x0, x5				//check if we have <= 8 blocks
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 3 - round 12
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 2 - round 12

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 6 - round 12
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 0 - round 12
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 7 - round 12

	aese	v5.16b, v27.16b						//AES block 5 - round 13
	aese	v1.16b, v27.16b						//AES block 1 - round 13
	aese	v2.16b, v27.16b						//AES block 2 - round 13

	aese	v0.16b, v27.16b						//AES block 0 - round 13
	aese	v4.16b, v27.16b						//AES block 4 - round 13
	aese	v6.16b, v27.16b						//AES block 6 - round 13

	aese	v3.16b, v27.16b						//AES block 3 - round 13
	aese	v7.16b, v27.16b						//AES block 7 - round 13
	b.ge	.L256_dec_tail						//handle tail

	ldp	q8, q9, [x0], #32			//AES block 0, 1 - load ciphertext

	ldp	q10, q11, [x0], #32			//AES block 2, 3 - load ciphertext

	ldp	q12, q13, [x0], #32			//AES block 4, 5 - load ciphertext

	ldp	q14, q15, [x0], #32			//AES block 6, 7 - load ciphertext
	cmp	x0, x5				//check if we have <= 8 blocks

.inst	0xce017121	//eor3 v1.16b, v9.16b, v1.16b, v28.16b				//AES block 1 - result
.inst	0xce007100	//eor3 v0.16b, v8.16b, v0.16b, v28.16b				//AES block 0 - result
	stp	q0, q1, [x2], #32			//AES block 0, 1 - store result

	rev32	v0.16b, v30.16b				//CTR block 8
	add	v30.4s, v30.4s, v31.4s		//CTR block 8
.inst	0xce037163	//eor3 v3.16b, v11.16b, v3.16b, v28.16b				//AES block 3 - result

.inst	0xce0571a5	//eor3 v5.16b, v13.16b, v5.16b, v28.16b				//AES block 5 - result

.inst	0xce047184	//eor3 v4.16b, v12.16b, v4.16b, v28.16b				//AES block 4 - result
	rev32	v1.16b, v30.16b				//CTR block 9
	add	v30.4s, v30.4s, v31.4s		//CTR block 9

.inst	0xce027142	//eor3 v2.16b, v10.16b, v2.16b, v28.16b				//AES block 2 - result
	stp	q2, q3, [x2], #32			//AES block 2, 3 - store result

	rev32	v2.16b, v30.16b				//CTR block 10
	add	v30.4s, v30.4s, v31.4s		//CTR block 10

.inst	0xce0671c6	//eor3 v6.16b, v14.16b, v6.16b, v28.16b				//AES block 6 - result

	rev32	v3.16b, v30.16b				//CTR block 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 11
	stp	q4, q5, [x2], #32			//AES block 4, 5 - store result

.inst	0xce0771e7	//eor3 v7.16b, v15.16b, v7.16b, v28.16b				//AES block 7 - result
	stp	q6, q7, [x2], #32			//AES block 6, 7 - store result

	rev32	v4.16b, v30.16b				//CTR block 12
	add	v30.4s, v30.4s, v31.4s		//CTR block 12
	b.ge	.L256_dec_prepretail					//do prepretail

.L256_dec_main_loop:	//main	loop start
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	ldp	q26, q27, [x8, #0]					//load rk0, rk1
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	rev64	v9.16b, v9.16b						//GHASH block 8k+1
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14
	rev64	v8.16b, v8.16b						//GHASH block 8k

	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0
	rev64	v12.16b, v12.16b						//GHASH block 8k+4
	rev64	v11.16b, v11.16b						//GHASH block 8k+3

	rev32	v7.16b, v30.16b				//CTR block 8k+15
	rev64	v15.16b, v15.16b						//GHASH block 8k+7

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0

	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	ldp	q28, q26, [x8, #32]				//load rk2, rk3

	eor	v8.16b, v8.16b, v19.16b					//PRE 1
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1

	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid

	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high

	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4

	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid
	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low

	ldp	q26, q27, [x8, #96]				//load rk6, rk7
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5

.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	rev64	v13.16b, v13.16b						//GHASH block 8k+5

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6

	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v14.16b, v14.16b						//GHASH block 8k+6
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7
	ldp	q28, q26, [x8, #128]				//load rk8, rk9

	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7

	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7

	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low
	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8

	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high

	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8

	ldp	q27, q28, [x8, #160]				//load rk10, rk11
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9

	ldp	q8, q9, [x0], #32			//AES block 8k+8, 8k+9 - load ciphertext
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9

	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9

	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high

	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10

	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9
.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid
.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
	rev32	v20.16b, v30.16b					//CTR block 8k+16
	ldr	d16, [x10]			//MODULO - load modulo constant

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+16
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 11
	ldp	q26, q27, [x8, #192]				//load rk12, rk13

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 11
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 11

.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid
	rev32	v22.16b, v30.16b					//CTR block 8k+17
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 11

	ldp	q10, q11, [x0], #32			//AES block 8k+10, 8k+11 - load ciphertext
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 11
	ext	v21.16b, v17.16b, v17.16b, #8				 //MODULO - other top alignment

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 11
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+17
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 11

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 12
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 12
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 12

	rev32	v23.16b, v30.16b					//CTR block 8k+18
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+18
	pmull	v29.1q, v17.1d, v16.1d			//MODULO - top 64b align with mid

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 12
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 11

	ldr	q28, [x8, #224]					//load rk14
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 12
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 12

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid
	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 12
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 12

	ldp	q12, q13, [x0], #32			//AES block 8k+12, 8k+13 - load ciphertext
	aese	v1.16b, v27.16b						//AES block 8k+9 - round 13
	aese	v2.16b, v27.16b						//AES block 8k+10 - round 13

	ldp	q14, q15, [x0], #32			//AES block 8k+14, 8k+15 - load ciphertext
	aese	v0.16b, v27.16b						//AES block 8k+8 - round 13
	aese	v5.16b, v27.16b						//AES block 8k+13 - round 13

	rev32	v25.16b, v30.16b					//CTR block 8k+19
.inst	0xce027142	//eor3 v2.16b, v10.16b, v2.16b, v28.16b				//AES block 8k+10 - result
.inst	0xce017121	//eor3 v1.16b, v9.16b, v1.16b, v28.16b				//AES block 8k+9 - result

	ext	v21.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment
	aese	v7.16b, v27.16b						//AES block 8k+15 - round 13

	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+19
	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v4.16b, v27.16b						//AES block 8k+12 - round 13

.inst	0xce0571a5	//eor3 v5.16b, v13.16b, v5.16b, v28.16b				//AES block 8k+13 - result
.inst	0xce007100	//eor3 v0.16b, v8.16b, v0.16b, v28.16b				//AES block 8k+8 - result
	aese	v3.16b, v27.16b						//AES block 8k+11 - round 13

	stp	q0, q1, [x2], #32			//AES block 8k+8, 8k+9 - store result
	mov	v0.16b, v20.16b					//CTR block 8k+16
.inst	0xce047184	//eor3 v4.16b, v12.16b, v4.16b, v28.16b				//AES block 8k+12 - result

.inst	0xce154673	//eor3 v19.16b, v19.16b, v21.16b, v17.16b		 	//MODULO - fold into low
.inst	0xce037163	//eor3 v3.16b, v11.16b, v3.16b, v28.16b				//AES block 8k+11 - result
	stp	q2, q3, [x2], #32			//AES block 8k+10, 8k+11 - store result

	mov	v3.16b, v25.16b					//CTR block 8k+19
	mov	v2.16b, v23.16b					//CTR block 8k+18
	aese	v6.16b, v27.16b						//AES block 8k+14 - round 13

	mov	v1.16b, v22.16b					//CTR block 8k+17
	stp	q4, q5, [x2], #32			//AES block 8k+12, 8k+13 - store result
.inst	0xce0771e7	//eor3 v7.16b, v15.16b, v7.16b, v28.16b				//AES block 8k+15 - result

.inst	0xce0671c6	//eor3 v6.16b, v14.16b, v6.16b, v28.16b				//AES block 8k+14 - result
	rev32	v4.16b, v30.16b				//CTR block 8k+20
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+20

	cmp	x0, x5				//.LOOP CONTROL
	stp	q6, q7, [x2], #32			//AES block 8k+14, 8k+15 - store result
	b.lt	.L256_dec_main_loop

.L256_dec_prepretail:	//PREPRETAIL
	ldp	q26, q27, [x8, #0]					//load rk0, rk1
	rev32	v5.16b, v30.16b				//CTR block 8k+13
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+13

	rev64	v12.16b, v12.16b						//GHASH block 8k+4
	ldr	q21, [x3, #144]				//load h6k | h5k
	ldr	q24, [x3, #192]				//load h8k | h7k

	rev32	v6.16b, v30.16b				//CTR block 8k+14
	rev64	v8.16b, v8.16b						//GHASH block 8k
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+14

	ext	v19.16b, v19.16b, v19.16b, #8				//PRE 0
	ldr	q23, [x3, #176]				//load h7l | h7h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #208]				//load h8l | h8h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v9.16b, v9.16b						//GHASH block 8k+1

	rev32	v7.16b, v30.16b				//CTR block 8k+15
	rev64	v10.16b, v10.16b						//GHASH block 8k+2
	ldr	q20, [x3, #128]				//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #160]				//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 0
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 0
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 0

	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 0
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 0
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 0

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 1
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 0
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 0

	ldp	q28, q26, [x8, #32]				//load rk2, rk3
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 1
	eor	v8.16b, v8.16b, v19.16b					//PRE 1

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 1
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 1
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 1

	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 1
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 1
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 1

	pmull2	v16.1q, v9.2d, v23.2d				//GHASH block 8k+1 - high
	trn1	v18.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	pmull	v19.1q, v8.1d, v25.1d				//GHASH block 8k - low

	rev64	v11.16b, v11.16b						//GHASH block 8k+3
	pmull	v23.1q, v9.1d, v23.1d				//GHASH block 8k+1 - low

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 2
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 2
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 2

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 2
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 2
	pmull2	v17.1q, v8.2d, v25.2d				//GHASH block 8k - high

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 2
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 3

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 3
	rev64	v14.16b, v14.16b						//GHASH block 8k+6

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 3
	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 2
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 3

	pmull2	v29.1q, v10.2d, v22.2d				//GHASH block 8k+2 - high
	trn2	v8.2d, v9.2d, v8.2d				//GHASH block 8k, 8k+1 - mid
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 2

	ldp	q27, q28, [x8, #64]				//load rk4, rk5
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 3
	pmull2	v9.1q, v11.2d, v20.2d				//GHASH block 8k+3 - high

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 3
	eor	v17.16b, v17.16b, v16.16b				//GHASH block 8k+1 - high
	eor	v8.16b, v8.16b, v18.16b			//GHASH block 8k, 8k+1 - mid

	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 3
	pmull	v22.1q, v10.1d, v22.1d				//GHASH block 8k+2 - low
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 3

.inst	0xce1d2631	//eor3 v17.16b, v17.16b, v29.16b, v9.16b			//GHASH block 8k+2, 8k+3 - high
	trn1	v29.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid
	trn2	v10.2d, v11.2d, v10.2d				//GHASH block 8k+2, 8k+3 - mid

	pmull2	v18.1q, v8.2d, v24.2d				//GHASH block 8k	- mid
	pmull	v20.1q, v11.1d, v20.1d				//GHASH block 8k+3 - low
	eor	v19.16b, v19.16b, v23.16b				//GHASH block 8k+1 - low

	pmull	v24.1q, v8.1d, v24.1d				//GHASH block 8k+1 - mid
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 4
	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 4

.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+2, 8k+3 - low
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8
	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 4

	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 4
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 4
	eor	v18.16b, v18.16b, v24.16b				//GHASH block 8k+1 - mid

	eor	v10.16b, v10.16b, v29.16b				//GHASH block 8k+2, 8k+3 - mid
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 5
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 4

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 5
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 4
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 4

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 5
	pmull2	v29.1q, v10.2d, v21.2d				//GHASH block 8k+2 - mid
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 5

	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 5
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 5
	pmull	v21.1q, v10.1d, v21.1d				//GHASH block 8k+3 - mid

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 5
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 5
	ldp	q26, q27, [x8, #96]				//load rk6, rk7

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v15.16b, v15.16b						//GHASH block 8k+7
	rev64	v13.16b, v13.16b						//GHASH block 8k+5

.inst	0xce157652	//eor3 v18.16b, v18.16b, v21.16b, v29.16b			//GHASH block 8k+2, 8k+3 - mid

	trn1	v16.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 6
	ldr	q21, [x3, #48]				//load h2k | h1k
	ldr	q24, [x3, #96]				//load h4k | h3k
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 6

	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 6
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 6

	pmull2	v8.1q, v12.2d, v25.2d				//GHASH block 8k+4 - high
	pmull2	v10.1q, v13.2d, v23.2d				//GHASH block 8k+5 - high
	pmull	v25.1q, v12.1d, v25.1d				//GHASH block 8k+4 - low

	trn2	v12.2d, v13.2d, v12.2d				//GHASH block 8k+4, 8k+5 - mid
	pmull	v23.1q, v13.1d, v23.1d				//GHASH block 8k+5 - low
	trn1	v13.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 7
	pmull2	v11.1q, v14.2d, v22.2d				//GHASH block 8k+6 - high
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 6

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 6
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 6
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 6

	ldp	q28, q26, [x8, #128]				//load rk8, rk9
	pmull	v22.1q, v14.1d, v22.1d				//GHASH block 8k+6 - low
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 7

	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 7
	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 7

	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 7
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 7
.inst	0xce082a31	//eor3 v17.16b, v17.16b, v8.16b, v10.16b			//GHASH block 8k+4, 8k+5 - high

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 7
	trn2	v14.2d, v15.2d, v14.2d				//GHASH block 8k+6, 8k+7 - mid
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 7

	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 8
	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 8
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 8

	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 8
	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 8
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 8

	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 8
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 9
	eor	v12.16b, v12.16b, v16.16b				//GHASH block 8k+4, 8k+5 - mid

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 9
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 9
	eor	v14.16b, v14.16b, v13.16b				//GHASH block 8k+6, 8k+7 - mid

	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 9
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 9
	pmull2	v16.1q, v12.2d, v24.2d				//GHASH block 8k+4 - mid

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 8
	pmull	v24.1q, v12.1d, v24.1d				//GHASH block 8k+5 - mid
	pmull2	v12.1q, v15.2d, v20.2d				//GHASH block 8k+7 - high

	pmull2	v13.1q, v14.2d, v21.2d				//GHASH block 8k+6 - mid
	pmull	v21.1q, v14.1d, v21.1d				//GHASH block 8k+7 - mid
	pmull	v20.1q, v15.1d, v20.1d				//GHASH block 8k+7 - low

	ldp	q27, q28, [x8, #160]				//load rk10, rk11
.inst	0xce195e73	//eor3 v19.16b, v19.16b, v25.16b, v23.16b			//GHASH block 8k+4, 8k+5 - low
.inst	0xce184252	//eor3 v18.16b, v18.16b, v24.16b, v16.16b			//GHASH block 8k+4, 8k+5 - mid

	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 9
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 9
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 9

.inst	0xce0b3231	//eor3 v17.16b, v17.16b, v11.16b, v12.16b			//GHASH block 8k+6, 8k+7 - high
.inst	0xce165273	//eor3 v19.16b, v19.16b, v22.16b, v20.16b			//GHASH block 8k+6, 8k+7 - low
	ldr	d16, [x10]			//MODULO - load modulo constant

.inst	0xce153652	//eor3 v18.16b, v18.16b, v21.16b, v13.16b			//GHASH block 8k+6, 8k+7 - mid

	aese	v4.16b, v27.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 10
	aese	v6.16b, v27.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 10
	aese	v5.16b, v27.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 10

	aese	v0.16b, v27.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 10
	aese	v2.16b, v27.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 10
	aese	v3.16b, v27.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 10

.inst	0xce114e52	//eor3 v18.16b, v18.16b, v17.16b, v19.16b		 	//MODULO - karatsuba tidy up

	aese	v7.16b, v27.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 10
	aese	v1.16b, v27.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 10
	ldp	q26, q27, [x8, #192]				//load rk12, rk13

	ext	v21.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment

	aese	v2.16b, v28.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 11
	aese	v1.16b, v28.16b
	aesmc	v1.16b, v1.16b			//AES block 8k+9 - round 11
	aese	v0.16b, v28.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 11

	pmull	v29.1q, v17.1d, v16.1d			//MODULO - top 64b align with mid
	aese	v3.16b, v28.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 11

	aese	v7.16b, v28.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 11
	aese	v6.16b, v28.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 11
	aese	v4.16b, v28.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 11

	aese	v5.16b, v28.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 11
	aese	v3.16b, v26.16b
	aesmc	v3.16b, v3.16b			//AES block 8k+11 - round 12

.inst	0xce1d5652	//eor3 v18.16b, v18.16b, v29.16b, v21.16b			//MODULO - fold into mid

	aese	v3.16b, v27.16b						//AES block 8k+11 - round 13
	aese	v2.16b, v26.16b
	aesmc	v2.16b, v2.16b			//AES block 8k+10 - round 12
	aese	v6.16b, v26.16b
	aesmc	v6.16b, v6.16b			//AES block 8k+14 - round 12

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low
	aese	v4.16b, v26.16b
	aesmc	v4.16b, v4.16b			//AES block 8k+12 - round 12
	aese	v7.16b, v26.16b
	aesmc	v7.16b, v7.16b			//AES block 8k+15 - round 12

	aese	v0.16b, v26.16b
	aesmc	v0.16b, v0.16b			//AES block 8k+8 - round 12
	ldr	q28, [x8, #224]					//load rk14
	aese	v1.16b, v26.16b
	aesmc	v1.16b, v1.16b	        	//AES block 8k+9 - round 12

	aese	v4.16b, v27.16b						//AES block 8k+12 - round 13
	ext	v21.16b, v18.16b, v18.16b, #8			 	//MODULO - other mid alignment
	aese	v5.16b, v26.16b
	aesmc	v5.16b, v5.16b			//AES block 8k+13 - round 12

	aese	v6.16b, v27.16b						//AES block 8k+14 - round 13
	aese	v2.16b, v27.16b						//AES block 8k+10 - round 13
	aese	v1.16b, v27.16b						//AES block 8k+9 - round 13

	aese	v5.16b, v27.16b						//AES block 8k+13 - round 13
.inst	0xce154673	//eor3 v19.16b, v19.16b, v21.16b, v17.16b		 	//MODULO - fold into low
	add	v30.4s, v30.4s, v31.4s		//CTR block 8k+15

	aese	v7.16b, v27.16b						//AES block 8k+15 - round 13
	aese	v0.16b, v27.16b						//AES block 8k+8 - round 13
.L256_dec_tail:	//TAIL

	ext	v16.16b, v19.16b, v19.16b, #8				//prepare final partial tag
	sub	x5, x4, x0		//main_end_input_ptr is number of bytes left to process
	cmp	x5, #112

	ldr	q9, [x0], #16				//AES block 8k+8 - load ciphertext

	ldp	q24, q25, [x3, #192]			//load h8k | h7k
	ext	v25.16b, v25.16b, v25.16b, #8
	mov	v29.16b, v28.16b

	ldp	q20, q21, [x3, #128]			//load h5l | h5h
	ext	v20.16b, v20.16b, v20.16b, #8

.inst	0xce00752c	//eor3 v12.16b, v9.16b, v0.16b, v29.16b				//AES block 8k+8 - result
	ldp	q22, q23, [x3, #160]			//load h6l | h6h
	ext	v22.16b, v22.16b, v22.16b, #8
	ext	v23.16b, v23.16b, v23.16b, #8
	b.gt	.L256_dec_blocks_more_than_7

	mov	v7.16b, v6.16b
	sub	v30.4s, v30.4s, v31.4s
	mov	v6.16b, v5.16b

	mov	v5.16b, v4.16b
	mov	v4.16b, v3.16b
	movi	v19.8b, #0

	movi	v17.8b, #0
	movi	v18.8b, #0
	mov	v3.16b, v2.16b

	cmp	x5, #96
	mov	v2.16b, v1.16b
	b.gt	.L256_dec_blocks_more_than_6

	mov	v7.16b, v6.16b
	mov	v6.16b, v5.16b

	mov	v5.16b, v4.16b
	cmp	x5, #80
	sub	v30.4s, v30.4s, v31.4s

	mov	v4.16b, v3.16b
	mov	v3.16b, v1.16b
	b.gt	.L256_dec_blocks_more_than_5

	cmp	x5, #64
	mov	v7.16b, v6.16b
	sub	v30.4s, v30.4s, v31.4s

	mov	v6.16b, v5.16b

	mov	v5.16b, v4.16b
	mov	v4.16b, v1.16b
	b.gt	.L256_dec_blocks_more_than_4

	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b
	cmp	x5, #48

	mov	v6.16b, v5.16b
	mov	v5.16b, v1.16b
	b.gt	.L256_dec_blocks_more_than_3

	ldr	q24, [x3, #96]				//load h4k | h3k
	sub	v30.4s, v30.4s, v31.4s
	mov	v7.16b, v6.16b

	cmp	x5, #32
	mov	v6.16b, v1.16b
	b.gt	.L256_dec_blocks_more_than_2

	sub	v30.4s, v30.4s, v31.4s

	mov	v7.16b, v1.16b
	cmp	x5, #16
	b.gt	.L256_dec_blocks_more_than_1

	sub	v30.4s, v30.4s, v31.4s
	ldr	q21, [x3, #48]				//load h2k | h1k
	b	.L256_dec_blocks_less_than_1
.L256_dec_blocks_more_than_7:	//blocks	left >  7
	rev64	v8.16b, v9.16b						//GHASH final-7 block
	ldr	q9, [x0], #16				//AES final-6 block - load ciphertext
	st1	{ v12.16b}, [x2], #16				//AES final-7 block  - store result

	ins	v18.d[0], v24.d[1]					//GHASH final-7 block - mid

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-7 block - mid
.inst	0xce01752c	//eor3 v12.16b, v9.16b, v1.16b, v29.16b				//AES final-6 block - result

	pmull2	v17.1q, v8.2d, v25.2d				//GHASH final-7 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-7 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull	v19.1q, v8.1d, v25.1d				//GHASH final-7 block - low
	pmull	v18.1q, v27.1d, v18.1d			 	//GHASH final-7 block - mid
.L256_dec_blocks_more_than_6:	//blocks	left >  6

	rev64	v8.16b, v9.16b						//GHASH final-6 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	ldr	q9, [x0], #16				//AES final-5 block - load ciphertext
	movi	v16.8b, #0						//supress further partial tag feed in

	ins	v27.d[0], v8.d[1]					//GHASH final-6 block - mid
	st1	{ v12.16b}, [x2], #16				//AES final-6 block - store result
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-6 block - high

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-6 block - low

.inst	0xce02752c	//eor3 v12.16b, v9.16b, v2.16b, v29.16b				//AES final-5 block - result
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-6 block - low
	eor	v27.8b, v27.8b, v8.8b				//GHASH final-6 block - mid

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-6 block - mid

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-6 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-6 block - high
.L256_dec_blocks_more_than_5:	//blocks	left >  5

	rev64	v8.16b, v9.16b						//GHASH final-5 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-5 block - high
	ins	v27.d[0], v8.d[1]					//GHASH final-5 block - mid

	ldr	q9, [x0], #16				//AES final-4 block - load ciphertext

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-5 block - mid
	st1	{ v12.16b}, [x2], #16			  	//AES final-5 block - store result

	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-5 block - low
	ins	v27.d[1], v27.d[0]					//GHASH final-5 block - mid

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-5 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-5 block - high
.inst	0xce03752c	//eor3 v12.16b, v9.16b, v3.16b, v29.16b				//AES final-4 block - result
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-5 block - low

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-5 block - mid
	movi	v16.8b, #0						//supress further partial tag feed in
.L256_dec_blocks_more_than_4:	//blocks	left >  4

	rev64	v8.16b, v9.16b						//GHASH final-4 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-4 block - mid
	ldr	q9, [x0], #16				//AES final-3 block - load ciphertext

	movi	v16.8b, #0						//supress further partial tag feed in

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final-4 block - low
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final-4 block - high

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-4 block - mid

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-4 block - high

	pmull	v27.1q, v27.1d, v21.1d				//GHASH final-4 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-4 block - low
	st1	{ v12.16b}, [x2], #16			 	//AES final-4 block - store result

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-4 block - mid
.inst	0xce04752c	//eor3 v12.16b, v9.16b, v4.16b, v29.16b				//AES final-3 block - result
.L256_dec_blocks_more_than_3:	//blocks	left >  3

	ldr	q25, [x3, #112]				//load h4l | h4h
	ext	v25.16b, v25.16b, v25.16b, #8
	rev64	v8.16b, v9.16b						//GHASH final-3 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag
	ldr	q9, [x0], #16				//AES final-2 block - load ciphertext
	ldr	q24, [x3, #96]				//load h4k | h3k

	ins	v27.d[0], v8.d[1]					//GHASH final-3 block - mid
	st1	{ v12.16b}, [x2], #16			 	//AES final-3 block - store result

.inst	0xce05752c	//eor3 v12.16b, v9.16b, v5.16b, v29.16b				//AES final-2 block - result

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-3 block - mid

	ins	v27.d[1], v27.d[0]					//GHASH final-3 block - mid
	pmull	v26.1q, v8.1d, v25.1d				//GHASH final-3 block - low
	pmull2	v28.1q, v8.2d, v25.2d				//GHASH final-3 block - high

	movi	v16.8b, #0						//supress further partial tag feed in
	pmull2	v27.1q, v27.2d, v24.2d				//GHASH final-3 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-3 block - low

	eor	v17.16b, v17.16b, v28.16b					//GHASH final-3 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-3 block - mid
.L256_dec_blocks_more_than_2:	//blocks	left >  2

	rev64	v8.16b, v9.16b						//GHASH final-2 block

	ldr	q23, [x3, #80]				//load h3l | h3h
	ext	v23.16b, v23.16b, v23.16b, #8
	ldr	q9, [x0], #16				//AES final-1 block - load ciphertext

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-2 block - mid

	pmull	v26.1q, v8.1d, v23.1d				//GHASH final-2 block - low
	st1	{ v12.16b}, [x2], #16			  	//AES final-2 block - store result
.inst	0xce06752c	//eor3 v12.16b, v9.16b, v6.16b, v29.16b				//AES final-1 block - result

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-2 block - mid
	eor	v19.16b, v19.16b, v26.16b					//GHASH final-2 block - low
	movi	v16.8b, #0						//supress further partial tag feed in

	pmull	v27.1q, v27.1d, v24.1d				//GHASH final-2 block - mid
	pmull2	v28.1q, v8.2d, v23.2d				//GHASH final-2 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-2 block - mid
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-2 block - high
.L256_dec_blocks_more_than_1:	//blocks	left >  1

	rev64	v8.16b, v9.16b						//GHASH final-1 block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v27.d[0], v8.d[1]					//GHASH final-1 block - mid
	ldr	q22, [x3, #64]				//load h2l | h2h
	ext	v22.16b, v22.16b, v22.16b, #8

	eor	v27.8b, v27.8b, v8.8b				//GHASH final-1 block - mid
	ldr	q9, [x0], #16				//AES final block - load ciphertext
	st1	{ v12.16b}, [x2], #16			 	//AES final-1 block - store result

	ldr	q21, [x3, #48]				//load h2k | h1k
	pmull	v26.1q, v8.1d, v22.1d				//GHASH final-1 block - low

	ins	v27.d[1], v27.d[0]					//GHASH final-1 block - mid

	eor	v19.16b, v19.16b, v26.16b					//GHASH final-1 block - low

.inst	0xce07752c	//eor3 v12.16b, v9.16b, v7.16b, v29.16b				//AES final block - result
	pmull2	v28.1q, v8.2d, v22.2d				//GHASH final-1 block - high

	pmull2	v27.1q, v27.2d, v21.2d				//GHASH final-1 block - mid

	movi	v16.8b, #0						//supress further partial tag feed in
	eor	v17.16b, v17.16b, v28.16b					//GHASH final-1 block - high

	eor	v18.16b, v18.16b, v27.16b				//GHASH final-1 block - mid
.L256_dec_blocks_less_than_1:	//blocks	left <= 1

	ld1	{ v26.16b}, [x2]					//load existing bytes where the possibly partial last block is to be stored
	mvn	x6, xzr						//temp0_x = 0xffffffffffffffff
	and	x1, x1, #127				//bit_length %= 128

	sub	x1, x1, #128				//bit_length -= 128
	rev32	v30.16b, v30.16b
	str	q30, [x16]					//store the updated counter

	neg	x1, x1				//bit_length = 128 - #bits in input (in range [1,128])

	and	x1, x1, #127			 	//bit_length %= 128

	lsr	x6, x6, x1				//temp0_x is mask for top 64b of last block
	cmp	x1, #64
	mvn	x7, xzr						//temp1_x = 0xffffffffffffffff

	csel	x14, x6, xzr, lt
	csel	x13, x7, x6, lt

	mov	v0.d[0], x13					//ctr0b is mask for last block
	mov	v0.d[1], x14

	and	v9.16b, v9.16b, v0.16b					//possibly partial last block has zeroes in highest bits
	ldr	q20, [x3, #32]				//load h1l | h1h
	ext	v20.16b, v20.16b, v20.16b, #8
	bif	v12.16b, v26.16b, v0.16b					//insert existing bytes in top end of result before storing

	rev64	v8.16b, v9.16b						//GHASH final block

	eor	v8.16b, v8.16b, v16.16b					//feed in partial tag

	ins	v16.d[0], v8.d[1]					//GHASH final block - mid
	pmull2	v28.1q, v8.2d, v20.2d				//GHASH final block - high

	eor	v16.8b, v16.8b, v8.8b				//GHASH final block - mid

	pmull	v26.1q, v8.1d, v20.1d				//GHASH final block - low
	eor	v17.16b, v17.16b, v28.16b					//GHASH final block - high

	pmull	v16.1q, v16.1d, v21.1d				//GHASH final block - mid

	eor	v18.16b, v18.16b, v16.16b				//GHASH final block - mid
	ldr	d16, [x10]			//MODULO - load modulo constant
	eor	v19.16b, v19.16b, v26.16b					//GHASH final block - low

	pmull	v21.1q, v17.1d, v16.1d		 	//MODULO - top 64b align with mid
	eor	v14.16b, v17.16b, v19.16b				//MODULO - karatsuba tidy up

	ext	v17.16b, v17.16b, v17.16b, #8				//MODULO - other top alignment
	st1	{ v12.16b}, [x2]				//store all 16B

	eor	v18.16b, v18.16b, v14.16b				//MODULO - karatsuba tidy up

	eor	v21.16b, v17.16b, v21.16b				//MODULO - fold into mid
	eor	v18.16b, v18.16b, v21.16b				//MODULO - fold into mid

	pmull	v17.1q, v18.1d, v16.1d			//MODULO - mid 64b align with low

	ext	v18.16b, v18.16b, v18.16b, #8				//MODULO - other mid alignment
	eor	v19.16b, v19.16b, v17.16b				//MODULO - fold into low

	eor	v19.16b, v19.16b, v18.16b				//MODULO - fold into low
	ext	v19.16b, v19.16b, v19.16b, #8
	rev64	v19.16b, v19.16b
	st1	{ v19.16b }, [x3]
	mov	x0, x9

	ldp	d10, d11, [sp, #16]
	ldp	d12, d13, [sp, #32]
	ldp	d14, d15, [sp, #48]
	ldp	d8, d9, [sp], #80
	ret

.L256_dec_ret:
	mov	w0, #0x0
	ret
.size	unroll8_eor3_aes_gcm_dec_256_kernel,.-unroll8_eor3_aes_gcm_dec_256_kernel
.byte	65,69,83,32,71,67,77,32,109,111,100,117,108,101,32,102,111,114,32,65,82,77,118,56,44,32,83,80,68,88,32,66,83,68,45,51,45,67,108,97,117,115,101,32,98,121,32,60,120,105,97,111,107,97,110,103,46,113,105,97,110,64,97,114,109,46,99,111,109,62,0
.align	2
.align	2
#endif
